COMPMID-2577: Fuse bias addition and activation in gemm assembly kernels
Change-Id: I7f52112d2d05b1ea3d3f3d4b19b8eafab05d6c44
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Reviewed-on: https://review.mlplatform.org/c/2141
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Marquez <pablo.tello@arm.com>
diff --git a/src/core/NEON/kernels/arm_gemm/bias_adder.hpp b/src/core/NEON/kernels/arm_gemm/bias_adder.hpp
new file mode 100644
index 0000000..745d005
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/bias_adder.hpp
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+namespace arm_gemm
+{
+
+// Fallback routine to add bias to a block
+template<typename T>
+inline void bias_adder(T *out, unsigned int stride, const T *bias, unsigned int rows, unsigned int cols) {
+ for (unsigned int row=0; row<rows; row++) {
+ for (unsigned int col=0; col<cols; col++) {
+ out[row * stride + col] += bias[col];
+ }
+ }
+}
+
+template<bool DoBias, typename T>
+inline void activator(T *out, unsigned int stride, const T *bias, Activation act, unsigned int rows, unsigned int cols) {
+ if (act.type == Activation::Type::None) {
+ if (DoBias) {
+ bias_adder(out, stride, bias, rows, cols);
+ }
+ return;
+ }
+
+ if (act.type == Activation::Type::ReLU) {
+ for (unsigned int row=0; row<rows; row++) {
+ for (unsigned int col=0; col<cols; col++) {
+ T &v = out[row * stride + col];
+ if (DoBias) {
+ v += bias[col];
+ }
+ v = std::max(static_cast<T>(0), v);
+ }
+ }
+ }
+
+ if (act.type == Activation::Type::BoundedReLU) {
+ const T max = static_cast<T>(act.param1);
+
+ for (unsigned int row=0; row<rows; row++) {
+ for (unsigned int col=0; col<cols; col++) {
+ T &v = out[row * stride + col];
+ if (DoBias) {
+ v += bias[col];
+ }
+ v = std::max(static_cast<T>(0), std::min(v, max));
+ }
+ }
+ }
+}
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
index 8541d34..b6671e8 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
@@ -47,23 +47,23 @@
{
GemmMethod::GEMM_HYBRID,
"hybrid_fp16_mla_4VLx4",
- [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; },
- [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<__fp16> &args) { return new GemmHybrid<hybrid_fp16_mla_4VLx4, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return (args._Ksize >= 8) && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp16_mla_4VLx4, __fp16, __fp16>(args); }
},
{
GemmMethod::GEMM_NATIVE,
"native_fp16_mla_4VLx4",
- [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8 && args._alpha==1.0f && !args._trA && !args._trB); },
- [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<__fp16> &args) { return new GemmNative<native_fp16_mla_4VLx4, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return (args._Ksize >= 8 && !args._trA && !args._trB); },
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmNative<native_fp16_mla_4VLx4, __fp16, __fp16>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
"interleaved_fp16_mla_3VLx8",
- [](const GemmArgs<__fp16> &args) { return (args._Ksize > 4); },
+ [](const GemmArgs &args) { return (args._Ksize > 4); },
nullptr,
- [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args); }
},
#endif
@@ -72,21 +72,21 @@
GemmMethod::GEMM_INTERLEAVED,
"hgemm_24x8",
#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
- [](const GemmArgs<__fp16> &args) { return args._ci->has_fp16(); },
+ [](const GemmArgs &args) { return args._ci->has_fp16(); },
#else
nullptr,
#endif
nullptr,
- [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args); }
},
-#endif
+#endif // aarch64 && FP16
#ifdef __aarch64__
{
GemmMethod::GEMM_INTERLEAVED,
"sgemm_12x8",
nullptr,
nullptr,
- [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<sgemm_12x8, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, __fp16, __fp16>(args); }
},
#elif defined(__arm__)
{
@@ -94,7 +94,7 @@
"sgemm_8x6",
nullptr,
nullptr,
- [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<sgemm_8x6, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<sgemm_8x6, __fp16, __fp16>(args); }
},
#else // not AArch64 or AArch32
# error Unknown Architecture
@@ -114,9 +114,9 @@
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &);
-template KernelDescription get_gemm_method<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &);
-template std::vector<KernelDescription> get_compatible_kernels<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &);
+template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &);
+template KernelDescription get_gemm_method<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &);
+template std::vector<KernelDescription> get_compatible_kernels<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
index dedcdb7..cf91ee0 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
@@ -33,17 +33,16 @@
#include "kernels/a32_sgemm_8x6.hpp"
#include "kernels/a64_hybrid_fp32_mla_16x4.hpp"
+#include "kernels/a64_native_fp32_mla_16x4.hpp"
+#include "kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp"
+#include "kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp"
#include "kernels/a64_sgemm_12x8.hpp"
-#include "kernels/a64_sgemm_native_16x4.hpp"
-#include "kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp"
#include "kernels/a64_sgemv_pretransposed.hpp"
#include "kernels/a64_sgemv_trans.hpp"
#include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp"
#include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp"
#include "kernels/sve_native_fp32_mla_4VLx4.hpp"
-#include "kernels/sve_smallK_fp32_mla_1VLx4.hpp"
-#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp"
namespace arm_gemm {
@@ -52,88 +51,81 @@
{
GemmMethod::GEMV_BATCHED,
"gemv_batched",
- [](const GemmArgs<float> &args) { return (args._Msize==1) && (args._nbatches>1); },
+ [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); },
nullptr,
- [](const GemmArgs<float> &args) { return new GemvBatched<float, float>(args); }
+ [](const GemmArgs &args) { return new GemvBatched<float, float>(args); }
},
#ifdef __aarch64__
{
GemmMethod::GEMV_PRETRANSPOSED,
"sgemv_pretransposed",
- [](const GemmArgs<float> &args) { return (args._Msize==1 && args._alpha==1.0f && args._pretransposed_hint && args._nbatches==1); },
+ [](const GemmArgs &args) { return (args._Msize==1 && args._pretransposed_hint && args._nbatches==1); },
nullptr,
- [](const GemmArgs<float> &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); }
+ [](const GemmArgs &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); }
},
{
GemmMethod::GEMV_NATIVE_TRANSPOSED,
"sgemv_trans",
- [](const GemmArgs<float> &args) { return (args._Msize==1 && args._alpha==1.0f && !args._trA && !args._trB && args._nbatches==1); },
+ [](const GemmArgs &args) { return (args._Msize==1 && !args._trA && !args._trB && args._nbatches==1); },
nullptr,
- [](const GemmArgs<float> &args) { return new GemvNativeTransposed<sgemv_trans, float, float>(args); }
+ [](const GemmArgs &args) { return new GemvNativeTransposed<sgemv_trans, float, float>(args); }
},
#ifdef __ARM_FEATURE_SVE
-// SVE smallk / native / hybrid methods
-{
- GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_fp32_mla_1VLx4",
- [](const GemmArgs<float> &args) { return (args._Ksize <= 24) && !args._trA && args._alpha==1.0f && args._pretransposed_hint; },
- nullptr,
- [](const GemmArgs<float> &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_1VLx4, float, float>(args); }
-},
+// SVE native / hybrid methods
{
GemmMethod::GEMM_HYBRID,
"hybrid_fp32_mla_4VLx4",
- [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; },
- [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<float> &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); }
-},
-{
- GemmMethod::GEMM_NATIVE,
- "smallK_fp32_mla_1VLx4",
- [](const GemmArgs<float> &args) { return (args._Ksize <= 24) && !args._trA && !args._trB && args._alpha==1.0f; },
- nullptr,
- [](const GemmArgs<float> &args) { return new GemmNative<smallK_fp32_mla_1VLx4, float, float>(args); }
+ [](const GemmArgs &args) { return (args._Ksize >= 4) && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); }
},
{
GemmMethod::GEMM_NATIVE,
"native_fp32_mla_4VLx4",
- [](const GemmArgs<float> &args) { return (args._Ksize>4 && args._alpha==1.0f && !args._trA && !args._trB); },
- [](const GemmArgs<float> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<float> &args) { return new GemmNative<native_fp32_mla_4VLx4, float, float>(args); }
+ [](const GemmArgs &args) { return (args._Ksize>4 && !args._trA && !args._trB); },
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmNative<native_fp32_mla_4VLx4, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE
// NEON native / hybrid methods
{
GemmMethod::GEMM_HYBRID,
- "sgemm_nativeA_pretransposeB_16x4",
- [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; },
- [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<float> &args) { return new GemmHybrid<sgemm_nativeA_pretransposeB_16x4, float, float>(args); }
+ "smallK_hybrid_fp32_mla_4x8",
+ [](const GemmArgs &args) { return (args._Ksize <= 8) && (args._Nsize % 4)==0 && !args._trA && args._pretransposed_hint; },
+ nullptr,
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x8, float, float>(args); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "smallK_hybrid_fp32_mla_4x6",
+ [](const GemmArgs &args) { return (args._Ksize > 8) && (args._Ksize <= 16) && (args._Nsize % 4)==0 && !args._trA && args._pretransposed_hint; },
+ nullptr,
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x6, float, float>(args); }
},
{
GemmMethod::GEMM_HYBRID,
"hybrid_fp32_mla_16x4",
- [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; },
- [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<float> &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); }
+ [](const GemmArgs &args) { return (args._Ksize >= 4) && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); }
},
{
GemmMethod::GEMM_NATIVE,
- "sgemm_native_16x4",
- [](const GemmArgs<float> &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && args._alpha==1.0f && !args._trA && !args._trB); },
- [](const GemmArgs<float> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<float> &args) { return new GemmNative<sgemm_native_16x4, float, float>(args); }
+ "native_fp32_mla_16x4",
+ [](const GemmArgs &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && !args._trA && !args._trB); },
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmNative<native_fp32_mla_16x4, float, float>(args); }
},
#ifdef __ARM_FEATURE_SVE
{
GemmMethod::GEMM_INTERLEAVED,
"interleaved_fp32_mla_3VLx8",
- [](const GemmArgs<float> &args) { return (args._Ksize>4); },
+ [](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs<float> &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE
{
@@ -141,7 +133,7 @@
"sgemm_12x8",
nullptr,
nullptr,
- [](const GemmArgs<float> &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); }
},
#endif // __aarch64__
@@ -151,7 +143,7 @@
"sgemm_8x6",
nullptr,
nullptr,
- [](const GemmArgs<float> &args) { return new GemmInterleaved<sgemm_8x6, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<sgemm_8x6, float, float>(args); }
},
#endif // __arm__
{
@@ -170,8 +162,8 @@
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<float, float> gemm<float, float, Nothing>(const GemmArgs<float> &args, const Nothing &);
-template KernelDescription get_gemm_method<float, float, Nothing>(const GemmArgs<float> &args, const Nothing &);
-template std::vector<KernelDescription> get_compatible_kernels<float, float, Nothing> (const GemmArgs<float> &args, const Nothing &);
+template UniqueGemmCommon<float, float> gemm<float, float, Nothing>(const GemmArgs &args, const Nothing &);
+template KernelDescription get_gemm_method<float, float, Nothing>(const GemmArgs &args, const Nothing &);
+template std::vector<KernelDescription> get_compatible_kernels<float, float, Nothing> (const GemmArgs &args, const Nothing &);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
index 436f55d..c3abb04 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -28,6 +28,7 @@
#include <algorithm>
#include "arm_gemm.hpp"
+#include "bias_adder.hpp"
#include "ndrange.hpp"
#include "utils.hpp"
@@ -58,7 +59,7 @@
const bool _trB;
- const Tr _beta;
+ const Activation _act;
/* Blocking info */
const unsigned int _k_block;
@@ -70,7 +71,12 @@
const NDRange<4> _window_range;
- static unsigned int compute_k_block(const GemmArgs<Tr> &args) {
+ static unsigned int compute_k_block(const GemmArgs &args) {
+ // Some kernels don't support append mode - these can't do K blocking at all.
+ if (!strategy::supports_append()) {
+ return args._Ksize;
+ }
+
if (args._cfg && args._cfg->inner_block_size) {
return args._cfg->inner_block_size;
}
@@ -97,7 +103,7 @@
return k_block;
}
- static unsigned int compute_n_block(const GemmArgs<Tr> &args) {
+ static unsigned int compute_n_block(const GemmArgs &args) {
if (args._cfg && args._cfg->outer_block_size) {
return args._cfg->outer_block_size;
}
@@ -127,9 +133,10 @@
GemmHybrid & operator= (GemmHybrid &) = delete;
/* Constructor */
- GemmHybrid(const GemmArgs<Tr> &args)
+ GemmHybrid(const GemmArgs &args)
: _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
- _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _beta(args._beta),
+ _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB),
+ _act(args._act),
_k_block(compute_k_block(args)), _n_block(compute_n_block(args)),
_Mround(roundup(args._Msize, strategy::out_height())),
_window_range(iceildiv(args._Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmulti) { }
@@ -164,6 +171,9 @@
unsigned int kmax = std::min(k0 + _k_block, _Ksize);
unsigned int kern_k = roundup(kmax-k0, strategy::k_unroll());
+ const bool first_pass = (k0 == 0);
+ const bool last_pass = (kmax == _Ksize);
+
auto p = _window_range.iterator(start, end);
if (p.done()) {
@@ -190,8 +200,17 @@
strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + k0, this->_lda,
b_panel,
this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc,
- (k0 == 0) ? _beta : static_cast<Tr>(1),
- (m_end - m_start), (nmax - n0), kmax-k0);
+ (m_end - m_start), (nmax - n0), kmax-k0,
+ (strategy::supports_bias() && first_pass && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr,
+ last_pass ? _act : Activation(), !first_pass);
+
+ // Add bias externally if needed
+ if (!strategy::supports_bias() && this->_bias && first_pass) {
+ bias_adder(this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc,
+ this->_bias + (multi * this->_bias_multi_stride) + n0,
+ (m_end - m_start), (nmax - n0));
+ }
+
} while (p.next_dim1());
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp
index b4edece..574ecef 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp
@@ -58,8 +58,6 @@
const bool _trB;
- const Tr _beta;
-
/* Blocking info */
const unsigned int _k_block;
const unsigned int _n_block;
@@ -82,7 +80,7 @@
return _Nsize * _nmulti * sizeof(int32_t);
}
- static unsigned int compute_k_block(const GemmArgs<Tr> &args) {
+ static unsigned int compute_k_block(const GemmArgs &args) {
// We don't support K blocks as we only temporarily store 32 bit results.
return args._Ksize;
@@ -112,7 +110,7 @@
return k_block;
}
- static unsigned int compute_n_block(const GemmArgs<Tr> &args) {
+ static unsigned int compute_n_block(const GemmArgs &args) {
if (args._cfg && args._cfg->outer_block_size) {
return args._cfg->outer_block_size;
}
@@ -142,9 +140,9 @@
GemmHybridQuantized & operator= (GemmHybridQuantized &) = delete;
/* Constructor */
- GemmHybridQuantized(const GemmArgs<Tr> &args, const ARequantizeLayer32 &qp)
+ GemmHybridQuantized(const GemmArgs &args, const ARequantizeLayer32 &qp)
: _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
- _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _beta(args._beta),
+ _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB),
_k_block(compute_k_block(args)), _n_block(compute_n_block(args)),
_Mround(roundup(args._Msize, strategy::out_height())),
_window_range(iceildiv(args._Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmulti),
@@ -210,8 +208,8 @@
strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + k0, this->_lda,
b_panel,
result_buffer, (nmax-n0),
- (k0 == 0) ? _beta : static_cast<Tr>(1),
- (m_end - m_start), (nmax - n0), kern_k);
+ (m_end - m_start), (nmax - n0), kern_k,
+ nullptr, Activation(), false);
}
{
@@ -262,7 +260,7 @@
col_bias = reinterpret_cast<int32_t *>(in_buffer);
for (unsigned int i=0; i<_nmulti; i++) {
- compute_col_sums(_qp, _Nsize, _Ksize, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize, 0);
+ compute_col_sums(_qp, _Nsize, _Ksize, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize, i, 0);
}
uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
@@ -295,8 +293,9 @@
col_bias = reinterpret_cast<int32_t *>(in_buffer);
}
- void set_quantized_bias(const int32_t *bias) override {
+ void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override {
_qp.bias = bias;
+ _qp.bias_multi_stride = bias_multi_stride;
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
index 55d72f8..569d1f4 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
@@ -34,13 +34,13 @@
*/
template<typename Top, typename Tret, class OutputStage = Nothing>
struct GemmImplementation {
- const GemmMethod method;
- const char * name;
- std::function<bool(const GemmArgs<Tret> &, const OutputStage &)> is_supported;
- std::function<bool(const GemmArgs<Tret> &, const OutputStage &)> is_recommended;
- std::function<GemmCommon<Top, Tret> *(const GemmArgs<Tret> &, const OutputStage &)> instantiate;
+ const GemmMethod method;
+ const char * name;
+ std::function<bool(const GemmArgs &, const OutputStage &)> is_supported;
+ std::function<bool(const GemmArgs &, const OutputStage &)> is_recommended;
+ std::function<GemmCommon<Top, Tret> *(const GemmArgs &, const OutputStage &)> instantiate;
- bool do_is_supported(const GemmArgs<Tret> &args, const OutputStage &os) const {
+ bool do_is_supported(const GemmArgs &args, const OutputStage &os) const {
if (is_supported != nullptr) {
return is_supported(args, os);
} else {
@@ -48,7 +48,7 @@
}
}
- bool do_is_recommended(const GemmArgs<Tret> &args, const OutputStage &os) const {
+ bool do_is_recommended(const GemmArgs &args, const OutputStage &os) const {
if (is_recommended != nullptr) {
return is_recommended(args, os);
} else {
@@ -56,7 +56,7 @@
}
}
- GemmCommon<Top, Tret> *do_instantiate(const GemmArgs<Tret> &args, const OutputStage &os) const {
+ GemmCommon<Top, Tret> *do_instantiate(const GemmArgs &args, const OutputStage &os) const {
return instantiate(args, os);
}
};
@@ -66,13 +66,13 @@
* unnecessary second argument. */
template<typename Top, typename Tret>
struct GemmImplementation<Top, Tret, Nothing> {
- const GemmMethod method;
- const char * name;
- std::function<bool(const GemmArgs<Tret> &)> is_supported;
- std::function<bool(const GemmArgs<Tret> &)> is_recommended;
- std::function<GemmCommon<Top, Tret> *(const GemmArgs<Tret> &)> instantiate;
+ const GemmMethod method;
+ const char * name;
+ std::function<bool(const GemmArgs &)> is_supported;
+ std::function<bool(const GemmArgs &)> is_recommended;
+ std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate;
- bool do_is_supported(const GemmArgs<Tret> &args, const Nothing &) const {
+ bool do_is_supported(const GemmArgs &args, const Nothing &) const {
if (is_supported != nullptr) {
return is_supported(args);
} else {
@@ -80,7 +80,7 @@
}
}
- bool do_is_recommended(const GemmArgs<Tret> &args, const Nothing &) const {
+ bool do_is_recommended(const GemmArgs &args, const Nothing &) const {
if (is_recommended != nullptr) {
return is_recommended(args);
} else {
@@ -88,7 +88,7 @@
}
}
- GemmCommon<Top, Tret> *do_instantiate(const GemmArgs<Tret> &args, const Nothing &) const {
+ GemmCommon<Top, Tret> *do_instantiate(const GemmArgs &args, const Nothing &) const {
return instantiate(args);
}
};
@@ -116,7 +116,7 @@
* reference.
*/
template<typename Top, typename Tret, class OutputStage>
-bool find_implementation(const GemmArgs<Tret> &args, const OutputStage &os, const GemmImplementation<Top, Tret, OutputStage> * &impl) {
+bool find_implementation(const GemmArgs &args, const OutputStage &os, const GemmImplementation<Top, Tret, OutputStage> * &impl) {
auto gemms = gemm_implementation_list<Top, Tret, OutputStage>();
const GemmConfig *cfg = args._cfg;
@@ -168,7 +168,7 @@
}
template<typename Top, typename Tret, class OutputStage>
-std::vector<KernelDescription> get_compatible_kernels(const GemmArgs<Tret> &args, const OutputStage &os) {
+std::vector<KernelDescription> get_compatible_kernels(const GemmArgs &args, const OutputStage &os) {
std::vector<KernelDescription> res;
/* Find out what the default implementation in so we can set the flag accordingly later. */
@@ -190,7 +190,7 @@
}
template<typename Top, typename Tret, class OutputStage>
-UniqueGemmCommon<Top, Tret> gemm(const GemmArgs<Tret> &args, const OutputStage &os) {
+UniqueGemmCommon<Top, Tret> gemm(const GemmArgs &args, const OutputStage &os) {
const GemmImplementation<Top, Tret, OutputStage> *impl;
if (find_implementation<Top, Tret, OutputStage>(args, os, impl)) {
@@ -201,7 +201,7 @@
}
template<typename Top, typename Tret, class OutputStage>
-KernelDescription get_gemm_method(const GemmArgs<Tret> &args, const OutputStage &os) {
+KernelDescription get_gemm_method(const GemmArgs &args, const OutputStage &os) {
const GemmImplementation<Top, Tret, OutputStage> *impl;
if (find_implementation<Top, Tret>(args, os, impl)) {
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
index 8062c30..e3b4416 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
@@ -38,7 +38,7 @@
"gemm_s16_12x8",
nullptr,
nullptr,
- [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s16_12x8, int16_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<gemm_s16_12x8, int16_t, int32_t>(args); }
},
{
GemmMethod::DEFAULT,
@@ -55,9 +55,9 @@
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<int16_t, int32_t> gemm<int16_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &);
-template KernelDescription get_gemm_method<int16_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &);
-template std::vector<KernelDescription> get_compatible_kernels<int16_t, int32_t, Nothing> (const GemmArgs<int32_t> &args, const Nothing &);
+template UniqueGemmCommon<int16_t, int32_t> gemm<int16_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template KernelDescription get_gemm_method<int16_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template std::vector<KernelDescription> get_compatible_kernels<int16_t, int32_t, Nothing> (const GemmArgs &args, const Nothing &);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
index 49241a7..a3446b9 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
@@ -48,66 +48,66 @@
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_s8s32_dot_1VLx8",
- [](const GemmArgs<int32_t> &args) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<int32_t> &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_1VLx8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_1VLx8, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
"hybrid_s8s32_dot_4VLx4",
- [](const GemmArgs<int32_t> &args) { return args._Ksize>=16 && args._alpha==1 && !args._trA && args._pretransposed_hint; },
- [](const GemmArgs<int32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<int32_t> &args) { return new GemmHybrid<hybrid_s8s32_dot_4VLx4, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return args._Ksize>=16 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmHybrid<hybrid_s8s32_dot_4VLx4, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_NATIVE,
"native_s8s32_dot_4VLx4",
- [](const GemmArgs<int32_t> &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); },
- [](const GemmArgs<int32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
- [](const GemmArgs<int32_t> &args) { return new GemmNative<native_s8s32_dot_4VLx4, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return (args._Ksize>=16 && !args._trA && !args._trB); },
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
+ [](const GemmArgs &args) { return new GemmNative<native_s8s32_dot_4VLx4, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
"interleaved_s8s32_dot_3VLx8",
- [](const GemmArgs<int32_t> &args) { return (args._Ksize>4) && args._alpha==1 && (args._beta == 0 || args._beta==1); },
+ [](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args); }
},
#endif
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_s8s32_dot_4x8",
- [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<int32_t> &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x8, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_s8s32_dot_4x6",
- [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<int32_t> &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x6, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x6, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
"hybrid_s8s32_dot_16x4",
- [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; },
- [](const GemmArgs<int32_t> &args) { return args._Nsize<=256 && args._Ksize>128; },
- [](const GemmArgs<int32_t> &args) { return new GemmHybrid<hybrid_s8s32_dot_16x4, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; },
+ [](const GemmArgs &args) { return new GemmHybrid<hybrid_s8s32_dot_16x4, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
"gemm_s8_12x8",
- [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && args._alpha==1 && (args._beta==0 || args._beta==1); },
+ [](const GemmArgs &args) { return args._ci->has_dotprod(); },
nullptr,
- [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
"gemm_s8_4x4",
nullptr,
nullptr,
- [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s8_4x4, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<gemm_s8_4x4, int8_t, int32_t>(args); }
},
{
GemmMethod::DEFAULT,
@@ -124,9 +124,9 @@
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<int8_t, int32_t> gemm<int8_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &);
-template KernelDescription get_gemm_method<int8_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &);
-template std::vector<KernelDescription> get_compatible_kernels<int8_t, int32_t, Nothing> (const GemmArgs<int32_t> &args, const Nothing &);
+template UniqueGemmCommon<int8_t, int32_t> gemm<int8_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template KernelDescription get_gemm_method<int8_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template std::vector<KernelDescription> get_compatible_kernels<int8_t, int32_t, Nothing> (const GemmArgs &args, const Nothing &);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
index faff9ac..efd9845 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
@@ -68,8 +68,7 @@
const bool _trA;
const bool _trB;
- const Tr _alpha;
- const Tr _beta;
+ const Activation _act;
const int _maxthreads;
int _nthreads;
@@ -297,9 +296,14 @@
#ifdef CYCLE_PROFILING
auto p=prof.ScopedProfiler(PROFILE_MERGE, (strategy::out_height() * bblocks * strategy::out_width() * sizeof(Tr)));
#endif
+ /* Only activate on last pass, only add bias on first pass, ask for accumulation on any non-first pass */
+ const bool first_pass = current.k0()==0;
+ const bool last_pass = current.kmax()==_Ksize;
+
strat.transforms.Merge(this->_Cptr + (batch * this->_C_batch_stride) + (current.multi() * this->_C_multi_stride),
c_panel, this->_ldc, y, ymax, current.x0(), current.xmax(),
- _alpha, (current.k0()==0 ? _beta : static_cast<Tr>(1)));
+ ((first_pass && this->_bias) ? this->_bias + (current.multi() * this->_bias_multi_stride) : nullptr),
+ (last_pass ? _act : Activation()), !first_pass);
}
}
}
@@ -317,10 +321,10 @@
GemmInterleaved & operator= (GemmInterleaved &) = delete;
/* Constructor */
- GemmInterleaved(const GemmArgs<Tr> &args)
+ GemmInterleaved(const GemmArgs &args)
: _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
_nbatches(args._nbatches), _nmulti(args._nmulti), _trA(args._trA), _trB(args._trB),
- _alpha(args._alpha), _beta(args._beta), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
+ _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
_pretransposed(args._pretransposed_hint) {
const unsigned int L1_size = _ci->get_L1_cache_size();
const unsigned int L2_size = _ci->get_L2_cache_size();
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp
index ba9163b..fe6ebef 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp
@@ -54,7 +54,7 @@
const unsigned int _nbatches;
const unsigned int _nmultis;
- const Tr _beta;
+ const Activation _act;
const CPUInfo * const _ci;
@@ -63,11 +63,11 @@
const NDRange<4> _window_range;
- static unsigned int compute_k_block(const GemmArgs<Tr> &args) {
+ static unsigned int compute_k_block(const GemmArgs &args) {
return args._Ksize;
}
- static unsigned int compute_n_block(const GemmArgs<Tr> &args) {
+ static unsigned int compute_n_block(const GemmArgs &args) {
if ((args._cfg != nullptr) && args._cfg->outer_block_size > 0) {
return args._cfg->outer_block_size;
} else {
@@ -79,10 +79,10 @@
GemmNative(GemmNative &) = delete;
GemmNative & operator= (GemmNative &) = delete;
- GemmNative(const GemmArgs<Tr> &args)
+ GemmNative(const GemmArgs &args)
: _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
_nbatches(args._nbatches), _nmultis(args._nmulti),
- _beta(args._beta), _ci(args._ci),
+ _act(args._act), _ci(args._ci),
_k_block(compute_k_block(args)), _n_block(compute_n_block(args)),
_window_range(iceildiv(_Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmultis) { }
@@ -127,7 +127,16 @@
strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (y0 * this->_lda), this->_lda,
this->_Bptr + (multi * this->_B_multi_stride) + n0, this->_ldb,
this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (y0 * this->_ldc) + n0, this->_ldc,
- _beta, (ymax-y0), (nmax - n0), _Ksize);
+ (ymax-y0), (nmax-n0), _Ksize,
+ (strategy::supports_bias() && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr,
+ _act, false);
+
+ // Add bias externally if needed
+ if (!strategy::supports_bias() && this->_bias) {
+ bias_adder(this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (y0 * this->_ldc) + n0, this->_ldc,
+ this->_bias + (multi * this->_bias_multi_stride) + n0,
+ (ymax - y0), (nmax - n0));
+ }
} while (p.next_dim1());
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp
index a402332..079c04a 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp
@@ -42,45 +42,45 @@
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
"smallK_hybrid_u8u32_dot_1VLx8",
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint8_t>(args, qp); }
},
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
"hybrid_u8u32_dot_4VLx4",
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; },
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_4VLx4, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; },
+ [](const GemmArgs &args, const ARequantizeLayer32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_4VLx4, uint8_t, uint8_t>(args, qp); }
},
#endif
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
"smallK_hybrid_u8u32_dot_4x8",
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint8_t>(args, qp); }
},
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
"smallK_hybrid_u8u32_dot_4x6",
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint8_t>(args, qp); }
},
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
"hybrid_u8u32_dot_16x4",
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; },
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._Nsize<=256 && args._Ksize>128; },
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_16x4, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; },
+ [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Nsize<=256 && args._Ksize>128; },
+ [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_16x4, uint8_t, uint8_t>(args, qp); }
},
{
GemmMethod::QUANTIZE_WRAPPER,
"quantized_wrapper",
nullptr,
nullptr,
- [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new QuantizeWrapper<uint8_t, uint8_t, uint32_t>(args, qp); }
+ [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new QuantizeWrapper<uint8_t, uint8_t, uint32_t>(args, qp); }
},
{
GemmMethod::DEFAULT,
@@ -96,9 +96,9 @@
return gemm_quint8_methods;
}
-template UniqueGemmCommon<uint8_t, uint8_t> gemm<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &os);
-template KernelDescription get_gemm_method<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &os);
-template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &os);
+template UniqueGemmCommon<uint8_t, uint8_t> gemm<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs &args, const ARequantizeLayer32 &os);
+template KernelDescription get_gemm_method<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs &args, const ARequantizeLayer32 &os);
+template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs &args, const ARequantizeLayer32 &os);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
index 408911b..85a8a67 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
@@ -38,7 +38,7 @@
"gemm_u16_12x8",
nullptr,
nullptr,
- [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u16_12x8, uint16_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<gemm_u16_12x8, uint16_t, uint32_t>(args); }
},
{
GemmMethod::DEFAULT,
@@ -55,9 +55,9 @@
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<uint16_t, uint32_t> gemm<uint16_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &);
-template KernelDescription get_gemm_method<uint16_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &);
-template std::vector<KernelDescription> get_compatible_kernels<uint16_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &);
+template UniqueGemmCommon<uint16_t, uint32_t> gemm<uint16_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template KernelDescription get_gemm_method<uint16_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template std::vector<KernelDescription> get_compatible_kernels<uint16_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
index f5a3072..aead814 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
@@ -48,66 +48,66 @@
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_u8u32_dot_1VLx8",
- [](const GemmArgs<uint32_t> &args) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
"hybrid_u8u32_dot_4VLx4",
- [](const GemmArgs<uint32_t> &args) { return args._Ksize>=16 && args._alpha==1 && !args._trA && args._pretransposed_hint; },
- [](const GemmArgs<uint32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<hybrid_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return args._Ksize>=16 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmHybrid<hybrid_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_NATIVE,
"native_u8u32_dot_4VLx4",
- [](const GemmArgs<uint32_t> &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); },
- [](const GemmArgs<uint32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
- [](const GemmArgs<uint32_t> &args) { return new GemmNative<native_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return (args._Ksize>=16 && !args._trA && !args._trB); },
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
+ [](const GemmArgs &args) { return new GemmNative<native_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
"interleaved_u8u32_dot_3VLx8",
- [](const GemmArgs<uint32_t> &args) { return (args._Ksize>4) && args._alpha==1 && (args._beta==0 || args._beta==1); },
+ [](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args); }
},
#endif
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_u8u32_dot_4x8",
- [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_u8u32_dot_4x6",
- [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; },
nullptr,
- [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
"hybrid_u8u32_dot_16x4",
- [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; },
- [](const GemmArgs<uint32_t> &args) { return args._Nsize<=256 && args._Ksize>128; },
- [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<hybrid_u8u32_dot_16x4, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; },
+ [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; },
+ [](const GemmArgs &args) { return new GemmHybrid<hybrid_u8u32_dot_16x4, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
"gemm_u8_12x8",
- [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && args._alpha==1 && (args._beta==0 || args._beta==1); },
+ [](const GemmArgs &args) { return args._ci->has_dotprod(); },
nullptr,
- [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u8_12x8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<gemm_u8_12x8, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
"gemm_u8_4x4",
nullptr,
nullptr,
- [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u8_4x4, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<gemm_u8_4x4, uint8_t, uint32_t>(args); }
},
{
GemmMethod::DEFAULT,
@@ -124,9 +124,9 @@
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<uint8_t, uint32_t> gemm<uint8_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &);
-template KernelDescription get_gemm_method<uint8_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &);
-template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint32_t, Nothing> (const GemmArgs<uint32_t> &args, const Nothing &);
+template UniqueGemmCommon<uint8_t, uint32_t> gemm<uint8_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template KernelDescription get_gemm_method<uint8_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &);
+template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint32_t, Nothing> (const GemmArgs &args, const Nothing &);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp
index 4453ee8..be2f561 100644
--- a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp
@@ -36,9 +36,9 @@
UniqueGemmCommon<To, Tr> _subgemm = nullptr;
public:
- GemvBatched(const GemmArgs<Tr> &args) {
+ GemvBatched(const GemmArgs &args) {
/* Just create a subgemm with batches->M */
- GemmArgs<Tr> newargs = args;
+ GemmArgs newargs = args;
newargs._Msize = args._nbatches;
newargs._nbatches = 1;
newargs._cfg = nullptr;
@@ -47,13 +47,15 @@
void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride,
const To *B, const int ldb, const int B_multi_stride,
- Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride) override {
- UNUSED(lda);
- UNUSED(ldc);
+ Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride,
+ const Tr *bias, const int bias_multi_stride) override {
/* A and C's batch stride becomes their new row stride. New batch stride is 0 as nbatches for subgemm is always 1. */
_subgemm->set_arrays(A, A_batch_stride, 0, A_multi_stride,
B, ldb, B_multi_stride,
- C, C_batch_stride, 0, C_multi_stride);
+ C, C_batch_stride, 0, C_multi_stride,
+ bias, bias_multi_stride);
+ UNUSED(lda);
+ UNUSED(ldc);
}
unsigned int get_window_size() const override {
diff --git a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp
index 55b1f9b..49681ec 100644
--- a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp
@@ -53,7 +53,7 @@
const unsigned int _nmultis;
- const Tr _beta;
+ const Activation _act;
const CPUInfo * const _ci;
@@ -64,8 +64,8 @@
GemvNativeTransposed(GemvNativeTransposed &) = delete;
GemvNativeTransposed & operator= (GemvNativeTransposed &) = delete;
- GemvNativeTransposed(const GemmArgs<Tr> &args)
- : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _beta(args._beta), _ci(args._ci) {
+ GemvNativeTransposed(const GemmArgs &args)
+ : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _act(args._act), _ci(args._ci) {
/* For now don't do any blocking. TODO: figure out if we should. */
m_block = _Ksize;
n_block = _Nsize;
@@ -111,7 +111,18 @@
strat.kernel(this->_Bptr + (multi * this->_B_multi_stride) + (m0 * this->_ldb) + n0,
this->_Aptr + (multi * this->_A_multi_stride) + m0,
this->_Cptr + (multi * this->_C_multi_stride) + n0,
- _beta, this->_ldb, (mmax-m0), (nmax-n0));
+ static_cast<Tr>(0), this->_ldb, (mmax-m0), (nmax-n0));
+
+ // Handle activation separately for now
+ if (this->_bias) {
+ activator<true>(this->_Cptr + (multi * this->_C_multi_stride) + n0, 0,
+ this->_bias + (multi * this->_bias_multi_stride) + n0,
+ _act, 1, (nmax-n0));
+ } else {
+ activator<false>(this->_Cptr + (multi * this->_C_multi_stride) + n0, 0,
+ static_cast<const Tr *>(nullptr),
+ _act, 1, (nmax-n0));
+ }
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
index 9206418..26fdfba 100644
--- a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
@@ -26,7 +26,7 @@
#include <stdio.h>
#include "arm_gemm.hpp"
-
+#include "bias_adder.hpp"
#include "mergeresults.hpp"
#include "transform.hpp"
@@ -53,7 +53,7 @@
const bool _trB;
- const Tr _beta;
+ const Activation _act;
const CPUInfo * const _ci;
@@ -68,8 +68,8 @@
GemvPretransposed(GemvPretransposed &) = delete;
GemvPretransposed & operator= (GemvPretransposed &) = delete;
- GemvPretransposed(const GemmArgs<Tr> &args)
- : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _trB(args._trB), _beta(args._beta), _ci(args._ci),
+ GemvPretransposed(const GemmArgs &args)
+ : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _trB(args._trB), _act(args._act), _ci(args._ci),
_buffer_per_multi(_Ksize * iceildiv(_Nsize, strategy::A_interleave()) * strategy::A_interleave()) {
/* For now don't do any blocking. TODO: figure out if we should. */
if (args._cfg && args._cfg->inner_block_size) {
@@ -128,7 +128,18 @@
(_Ksize * strategy::A_interleave()),
this->_Aptr + (multi * this->_A_multi_stride) + m0,
this->_Cptr + (multi * this->_C_multi_stride) + n,
- _beta, (mmax-m0), (nmax-n));
+ static_cast<Tr>(0), (mmax-m0), (nmax-n));
+
+ // Handle activation separately for now
+ if (this->_bias) {
+ activator<true>(this->_Cptr + (multi * this->_C_multi_stride) + n, 0,
+ this->_bias + (multi * this->_bias_multi_stride) + n,
+ _act, 1, (nmax-n));
+ } else {
+ activator<false>(this->_Cptr + (multi * this->_C_multi_stride) + n, 0,
+ static_cast<const Tr *>(nullptr),
+ _act, 1, (nmax-n));
+ }
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp
index cc205dc..941281f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp
@@ -61,7 +61,9 @@
kern_type kernel = a64_gemm_s8_12x8;
gemm_s8_12x8(const CPUInfo *ci) {
- if (ci->get_cpu_model() == CPUModel::A55r1) {
+ auto mod = ci->get_cpu_model();
+
+ if (mod == CPUModel::A55r1) {
kernel = a64_gemm_s8_12x8_a55r1;
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp
index 9032ba6..c228b8b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp
@@ -25,8 +25,6 @@
#ifdef __aarch64__
-#include "arm_gemm.hpp"
-
#include "../std_transforms_fixed.hpp"
namespace arm_gemm {
@@ -71,7 +69,9 @@
kern_type kernel = a64_gemm_u8_12x8;
gemm_u8_12x8(const CPUInfo *ci) {
- if (ci->get_cpu_model() == CPUModel::A55r1) {
+ auto mod = ci->get_cpu_model();
+
+ if (mod == CPUModel::A55r1) {
kernel = a64_gemm_u8_12x8_a55r1;
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp
index 5b850b7..283fea1 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp
@@ -25,8 +25,6 @@
#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
-#include "arm_gemm.hpp"
-
#include "../std_transforms_fixed.hpp"
namespace arm_gemm {
@@ -66,7 +64,9 @@
kern_type kernel = a64_hgemm_asimd_24x8;
hgemm_24x8(const CPUInfo *ci) {
- if (ci->get_cpu_model() == CPUModel::A55r1) {
+ auto model = ci->get_cpu_model();
+
+ if (model == CPUModel::A55r1) {
kernel = a64_hgemm_asimd_24x8_a55r1;
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp
index 5605939..1ce934d 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,8 +32,8 @@
{
// Actual kernel implementations
-void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, float, int, int, int);
-void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, float, int, int, int);
+void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
class hybrid_fp32_mla_16x4
{
@@ -41,10 +41,10 @@
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int);
+ typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -54,11 +54,26 @@
return 16;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
StdTransformsFixed<operand_type, result_type, 4, 16, 1> transforms = {};
// Default to the generic kernel
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp
index 7261761..e3325f7 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp
@@ -25,20 +25,41 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
+void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) {
const int K_stride = K;
const long loops_count = ((K + 4) / 8) - 1;
K -= loops_count * 8;
const long regs_count = (K / 4) - 1;
K -= (regs_count + 1) * 4;
const long blocks_count = K / 1;
+ float nullbias[16];
+ if (!append && !bias) {
+ memset(nullbias, 0, (16 * sizeof(float)));
+ }
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ const float * const minptr = &minval;
+ const float * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
for (int y=0; y<M; y+=4) {
const float * const a_ptr0_base = A + (y * lda);
@@ -48,7 +69,6 @@
for (int x0=0; x0<N; x0+=16ul) {
const long width = std::min((unsigned long)N-x0, 16ul);
- const float *betaptr = β
long loops = loops_count;
long regs = regs_count;
long blocks = blocks_count;
@@ -58,7 +78,7 @@
float result_buffer[64];
const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float);
float *c_ptr_real = c_ptr0;
- if (use_result_buffer && !beta0) {
+ if (use_result_buffer && append) {
for(int cy=0; cy<std::min(M-y, 4); cy++) {
for(unsigned int cx=0; cx<width; cx++) {
result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
@@ -68,6 +88,7 @@
if (use_result_buffer) {
c_ptr0 = result_buffer;
}
+ const float *biasptr = bias ? bias+x0 : nullbias;
switch(M-y) {
case 1:
@@ -76,17 +97,17 @@
"temploadreg1 .req X1\n"
"temploadreg2 .req X2\n"
"temploadreg3 .req X3\n"
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v18.4s, #0\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v19.4s, #0\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -95,21 +116,16 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -434,6 +450,16 @@
"fmla v19.4s, v11.4s, v0.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -444,7 +470,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -458,56 +484,47 @@
"temploadreg3 .req X5\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v20.16b, v16.16b\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "mov v21.16b, v17.16b\n"
"ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
+ "mov v22.16b, v18.16b\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v19.4s, #0\n"
+ "mov v23.16b, v19.16b\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v20.4s, #0\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v21.4s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v22.4s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v23.4s, #0\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "fmul v20.4s, v20.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v21.4s, v21.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "fmul v22.4s, v22.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "fmul v23.4s, v23.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -931,9 +948,27 @@
"fmla v23.4s, v11.4s, v1.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
"str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
"str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
"str q19, [%[c_ptr0], #0x30]\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
"str q20, [c_ptr1]\n"
@@ -947,7 +982,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
@@ -965,131 +1000,118 @@
"add c_ptr1, %[c_ptr0], %[ldc]\n"
"add a_ptr2, a_ptr1, %[lda]\n"
"add c_ptr2, c_ptr1, %[ldc]\n"
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v20.16b, v16.16b\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "mov v21.16b, v17.16b\n"
"ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
+ "mov v22.16b, v18.16b\n"
"ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
+ "mov v23.16b, v19.16b\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v20.4s, #0\n"
+ "mov v24.16b, v16.16b\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v21.4s, #0\n"
+ "mov v25.16b, v17.16b\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v22.4s, #0\n"
+ "mov v26.16b, v18.16b\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v23.4s, #0\n"
+ "mov v27.16b, v19.16b\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v24.4s, #0\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v25.4s, #0\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v27.4s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ldr d14, [%[b_ptr0], #0x60]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ins v14.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"add a_ptr2, a_ptr2, #0x10\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "fmul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "fmul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "fmul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "fmul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "fmul v24.4s, v24.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v25.4s, v25.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "fmul v26.4s, v26.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q2, [a_ptr2]\n"
- "fmul v27.4s, v27.4s, v15.4s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v14.d[1], temploadreg2\n"
"cbz %[loops], 2f\n"
"3:\n"
"fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
+ "ins v14.d[1], temploadreg2\n"
"fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
+ "ldr d15, [%[b_ptr0], #-0x10]\n"
"fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
+ "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
+ "ldr d4, [%[a_ptr0]]\n"
"fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr d5, [a_ptr1]\n"
+ "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
"fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
+ "ldr d5, [a_ptr1]\n"
"fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr d6, [a_ptr2]\n"
+ "ldr temploadreg1, [a_ptr1, #0x8]\n"
"fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
+ "ldr d6, [a_ptr2]\n"
"fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
+ "ldr temploadreg2, [a_ptr2, #0x8]\n"
"fmla v19.4s, v11.4s, v0.s[0]\n"
- "ins v4.d[1], temploadreg0\n"
+ "ldr d8, [%[b_ptr0]]\n"
"fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ "ins v4.d[1], temploadreg0\n"
"fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"fmla v16.4s, v12.4s, v0.s[1]\n"
- "ins v5.d[1], temploadreg1\n"
+ "ldr d9, [%[b_ptr0], #0x10]\n"
"fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ "ins v5.d[1], temploadreg1\n"
"fmla v24.4s, v12.4s, v2.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"fmla v17.4s, v13.4s, v0.s[1]\n"
- "ins v6.d[1], temploadreg2\n"
+ "ldr d10, [%[b_ptr0], #0x20]\n"
"fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "ins v6.d[1], temploadreg2\n"
"fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"fmla v18.4s, v14.4s, v0.s[1]\n"
- "ins v15.d[1], temploadreg3\n"
+ "ldr d11, [%[b_ptr0], #0x30]\n"
"fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "ins v15.d[1], temploadreg3\n"
"fmla v26.4s, v14.4s, v2.s[1]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
"subs %[loops], %[loops], #0x1\n"
"fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ "ins v8.d[1], temploadreg0\n"
"fmla v23.4s, v15.4s, v1.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"fmla v27.4s, v15.4s, v2.s[1]\n"
+ "ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
"fmla v16.4s, v8.4s, v0.s[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"fmla v20.4s, v8.4s, v1.s[2]\n"
@@ -1113,41 +1135,39 @@
"fmla v23.4s, v11.4s, v1.s[2]\n"
"ins v15.d[1], temploadreg3\n"
"fmla v27.4s, v11.4s, v2.s[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v24.4s, v12.4s, v2.s[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v25.4s, v13.4s, v2.s[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "fmla v26.4s, v14.4s, v2.s[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
"ins v8.d[1], temploadreg0\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
+ "fmla v27.4s, v15.4s, v2.s[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
+ "ins v9.d[1], temploadreg1\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla v16.4s, v8.4s, v4.s[0]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
+ "fmla v20.4s, v8.4s, v5.s[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
+ "fmla v24.4s, v8.4s, v6.s[0]\n"
"ins v10.d[1], temploadreg2\n"
- "add a_ptr1, a_ptr1, #0x20\n"
"fmla v17.4s, v9.4s, v4.s[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
"fmla v21.4s, v9.4s, v5.s[0]\n"
@@ -1163,70 +1183,71 @@
"fmla v19.4s, v11.4s, v4.s[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
"fmla v23.4s, v11.4s, v5.s[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
"ins v13.d[1], temploadreg1\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
+ "fmla v27.4s, v11.4s, v6.s[0]\n"
"ins v14.d[1], temploadreg2\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
"ldr d8, [%[b_ptr0]]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "fmla v20.4s, v12.4s, v5.s[1]\n"
"ins v0.d[1], temploadreg0\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
+ "fmla v24.4s, v12.4s, v6.s[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ins v1.d[1], temploadreg1\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
+ "fmla v21.4s, v13.4s, v5.s[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
+ "fmla v25.4s, v13.4s, v6.s[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
"ins v15.d[1], temploadreg3\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "ldr d2, [a_ptr2, #-0x10]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr temploadreg2, [a_ptr2, #-0x8]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "fmla v22.4s, v14.4s, v5.s[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
+ "fmla v26.4s, v14.4s, v6.s[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- "ins v2.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "ldr d1, [a_ptr1, #-0x10]\n"
+ "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "ldr temploadreg1, [a_ptr1, #-0x8]\n"
+ "fmla v27.4s, v15.4s, v6.s[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"fmla v16.4s, v8.4s, v4.s[2]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"fmla v20.4s, v8.4s, v5.s[2]\n"
- "ins v9.d[1], temploadreg1\n"
+ "ins v1.d[1], temploadreg1\n"
"fmla v24.4s, v8.4s, v6.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ "ldr d2, [a_ptr2, #-0x10]\n"
+ "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
+ "ldr temploadreg2, [a_ptr2, #-0x8]\n"
+ "ins v9.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "ins v2.d[1], temploadreg2\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "fmla v21.4s, v9.4s, v5.s[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "fmla v25.4s, v9.4s, v6.s[2]\n"
"ins v12.d[1], temploadreg0\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
"fmla v19.4s, v11.4s, v4.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
+ "ins v10.d[1], temploadreg2\n"
"fmla v23.4s, v11.4s, v5.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"fmla v27.4s, v11.4s, v6.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ "ins v13.d[1], temploadreg1\n"
"fmla v16.4s, v12.4s, v4.s[3]\n"
+ "ins v15.d[1], temploadreg3\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "ins v14.d[1], temploadreg2\n"
+ "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ "fmla v26.4s, v10.4s, v6.s[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
"fmla v20.4s, v12.4s, v5.s[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
@@ -1258,13 +1279,13 @@
"ins v11.d[1], temploadreg3\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
"b.ne 3b\n"
"2:\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
+ "ins v14.d[1], temploadreg2\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
+ "ldr d15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [c_ptr1]\n"
+ "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"prfm PSTL1KEEP, [c_ptr2]\n"
"ins v15.d[1], temploadreg3\n"
"cbz %[regs], 4f\n"
@@ -1554,13 +1575,39 @@
"fmla v27.4s, v11.4s, v2.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
"str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
"str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
+ "fmax v24.4s, v24.4s, v14.4s\n"
"str q19, [%[c_ptr0], #0x30]\n"
+ "fmax v25.4s, v25.4s, v14.4s\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
+ "fmax v26.4s, v26.4s, v14.4s\n"
"str q20, [c_ptr1]\n"
+ "fmin v24.4s, v24.4s, v15.4s\n"
+ "fmin v25.4s, v25.4s, v15.4s\n"
+ "fmax v27.4s, v27.4s, v14.4s\n"
"str q21, [c_ptr1, #0x10]\n"
+ "fmin v26.4s, v26.4s, v15.4s\n"
+ "fmin v27.4s, v27.4s, v15.4s\n"
"str q22, [c_ptr1, #0x20]\n"
"str q23, [c_ptr1, #0x30]\n"
"str q24, [c_ptr2]\n"
@@ -1576,7 +1623,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
);
break;
@@ -1599,89 +1646,72 @@
"add c_ptr2, c_ptr1, %[ldc]\n"
"add a_ptr3, a_ptr2, %[lda]\n"
"add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v20.16b, v16.16b\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "mov v21.16b, v17.16b\n"
"ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
+ "mov v22.16b, v18.16b\n"
"ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
+ "mov v23.16b, v19.16b\n"
"ldr q3, [a_ptr3]\n"
- "movi v20.4s, #0\n"
+ "mov v24.16b, v16.16b\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v21.4s, #0\n"
+ "mov v25.16b, v17.16b\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v22.4s, #0\n"
+ "mov v26.16b, v18.16b\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v23.4s, #0\n"
+ "mov v27.16b, v19.16b\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v24.4s, #0\n"
+ "mov v28.16b, v16.16b\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v25.4s, #0\n"
+ "mov v29.16b, v17.16b\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v26.4s, #0\n"
+ "mov v30.16b, v18.16b\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
+ "mov v31.16b, v19.16b\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v28.4s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v29.4s, #0\n"
- "ins v14.d[1], temploadreg2\n"
- "movi v30.4s, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "movi v31.4s, #0\n"
"add a_ptr2, a_ptr2, #0x10\n"
+ "ins v14.d[1], temploadreg2\n"
"add a_ptr3, a_ptr3, #0x10\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "fmul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "fmul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "fmul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "fmul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "fmul v24.4s, v24.4s, v15.4s\n"
"ldr q28, [c_ptr3]\n"
- "fmul v25.4s, v25.4s, v15.4s\n"
"ldr q29, [c_ptr3, #0x10]\n"
- "fmul v26.4s, v26.4s, v15.4s\n"
"ldr q30, [c_ptr3, #0x20]\n"
- "fmul v27.4s, v27.4s, v15.4s\n"
"ldr q31, [c_ptr3, #0x30]\n"
- "fmul v28.4s, v28.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v29.4s, v29.4s, v15.4s\n"
- "ldr q1, [a_ptr1]\n"
- "fmul v30.4s, v30.4s, v15.4s\n"
- "ldr q2, [a_ptr2]\n"
- "fmul v31.4s, v31.4s, v15.4s\n"
- "ldr q3, [a_ptr3]\n"
- "ldr q8, [%[b_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q1, [a_ptr1]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q2, [a_ptr2]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q3, [a_ptr3]\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -2303,16 +2333,50 @@
"fmla v31.4s, v11.4s, v3.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
"str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
"str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
+ "fmax v24.4s, v24.4s, v14.4s\n"
"str q19, [%[c_ptr0], #0x30]\n"
+ "fmax v25.4s, v25.4s, v14.4s\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
+ "fmax v26.4s, v26.4s, v14.4s\n"
"str q20, [c_ptr1]\n"
+ "fmin v24.4s, v24.4s, v15.4s\n"
+ "fmin v25.4s, v25.4s, v15.4s\n"
+ "fmax v27.4s, v27.4s, v14.4s\n"
"str q21, [c_ptr1, #0x10]\n"
+ "fmin v26.4s, v26.4s, v15.4s\n"
+ "fmax v28.4s, v28.4s, v14.4s\n"
+ "fmax v29.4s, v29.4s, v14.4s\n"
"str q22, [c_ptr1, #0x20]\n"
+ "fmin v27.4s, v27.4s, v15.4s\n"
+ "fmax v30.4s, v30.4s, v14.4s\n"
+ "fmin v28.4s, v28.4s, v15.4s\n"
"str q23, [c_ptr1, #0x30]\n"
+ "fmin v29.4s, v29.4s, v15.4s\n"
+ "fmax v31.4s, v31.4s, v14.4s\n"
+ "fmin v30.4s, v30.4s, v15.4s\n"
"str q24, [c_ptr2]\n"
+ "fmin v31.4s, v31.4s, v15.4s\n"
"str q25, [c_ptr2, #0x10]\n"
"str q26, [c_ptr2, #0x20]\n"
"str q27, [c_ptr2, #0x30]\n"
@@ -2331,7 +2395,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
);
break;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp
index 504769b..d60d08a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp
@@ -25,20 +25,41 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
+void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) {
const int K_stride = K;
const long loops_count = ((K + 4) / 8) - 1;
K -= loops_count * 8;
const long regs_count = (K / 4) - 1;
K -= (regs_count + 1) * 4;
const long blocks_count = K / 1;
+ float nullbias[16];
+ if (!append && !bias) {
+ memset(nullbias, 0, (16 * sizeof(float)));
+ }
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ const float * const minptr = &minval;
+ const float * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
for (int y=0; y<M; y+=4) {
const float * const a_ptr0_base = A + (y * lda);
@@ -48,7 +69,6 @@
for (int x0=0; x0<N; x0+=16ul) {
const long width = std::min((unsigned long)N-x0, 16ul);
- const float *betaptr = β
long loops = loops_count;
long regs = regs_count;
long blocks = blocks_count;
@@ -58,7 +78,7 @@
float result_buffer[64];
const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float);
float *c_ptr_real = c_ptr0;
- if (use_result_buffer && !beta0) {
+ if (use_result_buffer && append) {
for(int cy=0; cy<std::min(M-y, 4); cy++) {
for(unsigned int cx=0; cx<width; cx++) {
result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
@@ -68,21 +88,22 @@
if (use_result_buffer) {
c_ptr0 = result_buffer;
}
+ const float *biasptr = bias ? bias+x0 : nullbias;
switch(M-y) {
case 1:
__asm __volatile (
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v18.4s, #0\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v19.4s, #0\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
@@ -90,21 +111,16 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
@@ -291,13 +307,23 @@
"fmla v19.4s, v11.4s, v0.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
"str q19, [%[c_ptr0], #0x30]\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
);
break;
@@ -307,55 +333,46 @@
"c_ptr1 .req X1\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v20.16b, v16.16b\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "mov v21.16b, v17.16b\n"
"ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
+ "mov v22.16b, v18.16b\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v19.4s, #0\n"
+ "mov v23.16b, v19.16b\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v20.4s, #0\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v21.4s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v22.4s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v23.4s, #0\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "fmul v20.4s, v20.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v21.4s, v21.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "fmul v22.4s, v22.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "fmul v23.4s, v23.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
@@ -635,9 +652,27 @@
"fmla v23.4s, v11.4s, v1.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
"str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
"str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
"str q19, [%[c_ptr0], #0x30]\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
"str q20, [c_ptr1]\n"
@@ -647,7 +682,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
);
break;
@@ -661,71 +696,58 @@
"add c_ptr1, %[c_ptr0], %[ldc]\n"
"add a_ptr2, a_ptr1, %[lda]\n"
"add c_ptr2, c_ptr1, %[ldc]\n"
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v20.16b, v16.16b\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "mov v21.16b, v17.16b\n"
"ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
+ "mov v22.16b, v18.16b\n"
"ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
+ "mov v23.16b, v19.16b\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v20.4s, #0\n"
+ "mov v24.16b, v16.16b\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v21.4s, #0\n"
+ "mov v25.16b, v17.16b\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v22.4s, #0\n"
+ "mov v26.16b, v18.16b\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v23.4s, #0\n"
+ "mov v27.16b, v19.16b\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v24.4s, #0\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v25.4s, #0\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v26.4s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
"add a_ptr1, a_ptr1, #0x10\n"
"add a_ptr2, a_ptr2, #0x10\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "fmul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "fmul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "fmul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "fmul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "fmul v24.4s, v24.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v25.4s, v25.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "fmul v26.4s, v26.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q2, [a_ptr2]\n"
- "fmul v27.4s, v27.4s, v15.4s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
@@ -1098,13 +1120,39 @@
"fmla v27.4s, v11.4s, v2.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
"str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
"str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
+ "fmax v24.4s, v24.4s, v14.4s\n"
"str q19, [%[c_ptr0], #0x30]\n"
+ "fmax v25.4s, v25.4s, v14.4s\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
+ "fmax v26.4s, v26.4s, v14.4s\n"
"str q20, [c_ptr1]\n"
+ "fmin v24.4s, v24.4s, v15.4s\n"
+ "fmin v25.4s, v25.4s, v15.4s\n"
+ "fmax v27.4s, v27.4s, v14.4s\n"
"str q21, [c_ptr1, #0x10]\n"
+ "fmin v26.4s, v26.4s, v15.4s\n"
+ "fmin v27.4s, v27.4s, v15.4s\n"
"str q22, [c_ptr1, #0x20]\n"
"str q23, [c_ptr1, #0x30]\n"
"str q24, [c_ptr2]\n"
@@ -1116,7 +1164,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -1135,87 +1183,70 @@
"add c_ptr2, c_ptr1, %[ldc]\n"
"add a_ptr3, a_ptr2, %[lda]\n"
"add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[beta0], 1f\n"
- "movi v16.4s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v20.16b, v16.16b\n"
"ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
+ "mov v21.16b, v17.16b\n"
"ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
+ "mov v22.16b, v18.16b\n"
"ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
+ "mov v23.16b, v19.16b\n"
"ldr q3, [a_ptr3]\n"
- "movi v20.4s, #0\n"
+ "mov v24.16b, v16.16b\n"
"ldr q8, [%[b_ptr0]]\n"
- "movi v21.4s, #0\n"
+ "mov v25.16b, v17.16b\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v22.4s, #0\n"
+ "mov v26.16b, v18.16b\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v23.4s, #0\n"
+ "mov v27.16b, v19.16b\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v24.4s, #0\n"
+ "mov v28.16b, v16.16b\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v25.4s, #0\n"
+ "mov v29.16b, v17.16b\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v26.4s, #0\n"
+ "mov v30.16b, v18.16b\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
+ "mov v31.16b, v19.16b\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v28.4s, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "movi v29.4s, #0\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "movi v30.4s, #0\n"
"add a_ptr3, a_ptr3, #0x10\n"
- "movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "fmul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "fmul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "fmul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "fmul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "fmul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "fmul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "fmul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "fmul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "fmul v24.4s, v24.4s, v15.4s\n"
"ldr q28, [c_ptr3]\n"
- "fmul v25.4s, v25.4s, v15.4s\n"
"ldr q29, [c_ptr3, #0x10]\n"
- "fmul v26.4s, v26.4s, v15.4s\n"
"ldr q30, [c_ptr3, #0x20]\n"
- "fmul v27.4s, v27.4s, v15.4s\n"
"ldr q31, [c_ptr3, #0x30]\n"
- "fmul v28.4s, v28.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "fmul v29.4s, v29.4s, v15.4s\n"
- "ldr q1, [a_ptr1]\n"
- "fmul v30.4s, v30.4s, v15.4s\n"
- "ldr q2, [a_ptr2]\n"
- "fmul v31.4s, v31.4s, v15.4s\n"
- "ldr q3, [a_ptr3]\n"
- "ldr q8, [%[b_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q1, [a_ptr1]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q2, [a_ptr2]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q3, [a_ptr3]\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
@@ -1681,16 +1712,50 @@
"fmla v31.4s, v11.4s, v3.s[0]\n"
"b.ne 7b\n"
"6:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
"str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
"str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
"str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
+ "fmax v24.4s, v24.4s, v14.4s\n"
"str q19, [%[c_ptr0], #0x30]\n"
+ "fmax v25.4s, v25.4s, v14.4s\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
+ "fmax v26.4s, v26.4s, v14.4s\n"
"str q20, [c_ptr1]\n"
+ "fmin v24.4s, v24.4s, v15.4s\n"
+ "fmin v25.4s, v25.4s, v15.4s\n"
+ "fmax v27.4s, v27.4s, v14.4s\n"
"str q21, [c_ptr1, #0x10]\n"
+ "fmin v26.4s, v26.4s, v15.4s\n"
+ "fmax v28.4s, v28.4s, v14.4s\n"
+ "fmax v29.4s, v29.4s, v14.4s\n"
"str q22, [c_ptr1, #0x20]\n"
+ "fmin v27.4s, v27.4s, v15.4s\n"
+ "fmax v30.4s, v30.4s, v14.4s\n"
+ "fmin v28.4s, v28.4s, v15.4s\n"
"str q23, [c_ptr1, #0x30]\n"
+ "fmin v29.4s, v29.4s, v15.4s\n"
+ "fmax v31.4s, v31.4s, v14.4s\n"
+ "fmin v30.4s, v30.4s, v15.4s\n"
"str q24, [c_ptr2]\n"
+ "fmin v31.4s, v31.4s, v15.4s\n"
"str q25, [c_ptr2, #0x10]\n"
"str q26, [c_ptr2, #0x20]\n"
"str q27, [c_ptr2, #0x30]\n"
@@ -1705,7 +1770,7 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp
index c8934df..5a6fabc 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,8 +32,8 @@
{
// Actual kernel implementations
-void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
-void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
class hybrid_s8s32_dot_16x4
{
@@ -41,10 +41,10 @@
typedef int8_t operand_type;
typedef int32_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+ typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -54,11 +54,26 @@
return 16;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {};
// Default to the generic kernel
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp
index 3f72e4b..3ecf015 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,14 +25,17 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0);
+void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) {
+ UNUSED(bias);
+ UNUSED(act);
+
const int K_stride = ((K + 3) / 4) * 4;
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
@@ -49,7 +52,6 @@
for (int x0=0; x0<N; x0+=16ul) {
const long width = std::min((unsigned long)N-x0, 16ul);
- const int32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long blocks = blocks_count;
@@ -60,7 +62,7 @@
int32_t result_buffer[64];
const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t);
int32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && !beta0) {
+ if (use_result_buffer && append) {
for(int cy=0; cy<std::min(M-y, 4); cy++) {
for(unsigned int cx=0; cx<width; cx++) {
result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
@@ -78,7 +80,6 @@
"temploadreg1 .req X1\n"
"temploadreg2 .req X2\n"
"temploadreg3 .req X3\n"
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -94,44 +95,21 @@
"ldr d14, [%[b_ptr0], #0x60]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -146,108 +124,108 @@
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
"ldr d8, [%[b_ptr0]]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ins v0.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
@@ -258,29 +236,29 @@
"ins v11.d[1], temploadreg3\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"ins v14.d[1], temploadreg2\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
@@ -291,137 +269,137 @@
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"ldr d8, [%[b_ptr0]]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -430,30 +408,30 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -464,7 +442,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -476,178 +454,144 @@
"temploadreg1 .req X3\n"
"temploadreg2 .req X4\n"
"temploadreg3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
"movi v18.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
"movi v19.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
"movi v20.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"movi v21.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
"movi v22.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
"movi v23.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"ldr q1, [a_ptr1]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "subs %[loops], %[loops], #0x1\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ "subs %[loops], %[loops], #0x1\n"
"ins v15.d[1], temploadreg3\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
"ldr d1, [a_ptr1, #-0x10]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
"ldr d8, [%[b_ptr0]]\n"
"ins v0.d[1], temploadreg0\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ins v1.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
@@ -656,55 +600,55 @@
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
@@ -714,223 +658,223 @@
"ins v11.d[1], temploadreg3\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
"ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0]]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
+ "prfm PSTL1KEEP, [c_ptr1]\n"
"ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "b 5f\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr d8, [%[b_ptr0]]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ "ldr d9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ "ldr d10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "ldr d11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ "ins v8.d[1], temploadreg0\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ "ldr d12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ "ldr d13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ "ins v9.d[1], temploadreg1\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ "ldr d14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ "ins v10.d[1], temploadreg2\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ "ldr d15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ "ins v11.d[1], temploadreg3\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ "ins v12.d[1], temploadreg0\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ "ins v13.d[1], temploadreg1\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ "ins v14.d[1], temploadreg2\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ "ins v15.d[1], temploadreg3\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -939,43 +883,43 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[0], [a_ptr1], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[1], [a_ptr1], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
"ld1 {v1.b}[2], [a_ptr1]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -992,7 +936,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
@@ -1006,228 +950,183 @@
"temploadreg1 .req X5\n"
"temploadreg2 .req X6\n"
"temploadreg3 .req X7\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
"movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
"movi v19.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
"movi v20.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"movi v21.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
"movi v22.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
"movi v23.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
+ "ldr d14, [%[b_ptr0], #0x60]\n"
"movi v24.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"movi v25.4s, #0\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v27.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ins v14.d[1], temploadreg2\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
- "ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
"ldr q1, [a_ptr1]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
- "ldr q2, [a_ptr2]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "movi v27.4s, #0\n"
"ins v14.d[1], temploadreg2\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "ldr q2, [a_ptr2]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
"ldr d1, [a_ptr1, #-0x10]\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ins v0.d[1], temploadreg0\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ins v1.d[1], temploadreg1\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"add a_ptr2, a_ptr2, #0x20\n"
@@ -1235,65 +1134,65 @@
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
"ldr d2, [a_ptr2, #-0x10]\n"
"prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
"ins v2.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
@@ -1304,275 +1203,275 @@
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"ins v14.d[1], temploadreg2\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"prfm PSTL1KEEP, [c_ptr1]\n"
"prfm PSTL1KEEP, [c_ptr2]\n"
"ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -1581,56 +1480,56 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[0], [a_ptr1], #1\n"
"ld1 {v2.b}[0], [a_ptr2], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[1], [a_ptr1], #1\n"
"ld1 {v2.b}[1], [a_ptr2], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
"ld1 {v1.b}[2], [a_ptr1]\n"
"ld1 {v2.b}[2], [a_ptr2]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -1653,7 +1552,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
);
break;
@@ -1670,364 +1569,309 @@
"temploadreg1 .req X7\n"
"temploadreg2 .req X8\n"
"temploadreg3 .req X9\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
"movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
"movi v19.4s, #0\n"
- "ldr q3, [a_ptr3]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
"movi v20.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"movi v21.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
"movi v22.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
"movi v23.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr d14, [%[b_ptr0], #0x60]\n"
"movi v24.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"movi v25.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"movi v26.4s, #0\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v28.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v29.4s, #0\n"
- "ins v14.d[1], temploadreg2\n"
- "movi v30.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "movi v31.4s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
- "ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
- "ldr q28, [c_ptr3]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "mul v28.4s, v28.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v29.4s, v29.4s, v15.4s\n"
"ldr q1, [a_ptr1]\n"
- "mul v30.4s, v30.4s, v15.4s\n"
+ "movi v27.4s, #0\n"
+ "ins v14.d[1], temploadreg2\n"
+ "movi v28.4s, #0\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "movi v29.4s, #0\n"
"ldr q2, [a_ptr2]\n"
- "mul v31.4s, v31.4s, v15.4s\n"
+ "movi v30.4s, #0\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "movi v31.4s, #0\n"
"ldr q3, [a_ptr3]\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
"add a_ptr3, a_ptr3, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v14.d[1], temploadreg2\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d7, [a_ptr3]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
"ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ins v7.d[1], temploadreg3\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
- ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
+ ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
"ins v0.d[1], temploadreg0\n"
- ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
+ ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
"ldr d1, [a_ptr1, #-0x10]\n"
- ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
+ ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
"ins v1.d[1], temploadreg1\n"
- ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
+ ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
"add a_ptr2, a_ptr2, #0x20\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
"ldr d2, [a_ptr2, #-0x10]\n"
- ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
"ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
"ins v2.d[1], temploadreg2\n"
- ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
"add a_ptr3, a_ptr3, #0x20\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
"ldr d3, [a_ptr3, #-0x10]\n"
- ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
+ ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
"ldr temploadreg3, [a_ptr3, #-0x8]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
"ins v3.d[1], temploadreg3\n"
- ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
+ ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
+ ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
+ ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
+ ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
+ ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
+ ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
+ ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
"ins v14.d[1], temploadreg2\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
@@ -2035,319 +1879,319 @@
"prfm PSTL1KEEP, [c_ptr2]\n"
"prfm PSTL1KEEP, [c_ptr3]\n"
"ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"ldr d7, [a_ptr3]\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
"ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
"ins v7.d[1], temploadreg3\n"
- ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
+ ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
"add a_ptr3, a_ptr3, #0x10\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
+ ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
+ ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
+ ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -2356,69 +2200,69 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr s3, [a_ptr3]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"add a_ptr3, a_ptr3, #0x4\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[0], [a_ptr1], #1\n"
"ld1 {v2.b}[0], [a_ptr2], #1\n"
"ld1 {v3.b}[0], [a_ptr3], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[1], [a_ptr1], #1\n"
"ld1 {v2.b}[1], [a_ptr2], #1\n"
"ld1 {v3.b}[1], [a_ptr3], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
"ld1 {v1.b}[2], [a_ptr1]\n"
"ld1 {v2.b}[2], [a_ptr2]\n"
"ld1 {v3.b}[2], [a_ptr3]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -2447,7 +2291,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
);
break;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp
index 1ad9246..b48b674 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,14 +25,17 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0);
+void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) {
+ UNUSED(bias);
+ UNUSED(act);
+
const int K_stride = ((K + 3) / 4) * 4;
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
@@ -49,7 +52,6 @@
for (int x0=0; x0<N; x0+=16ul) {
const long width = std::min((unsigned long)N-x0, 16ul);
- const int32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long blocks = blocks_count;
@@ -60,7 +62,7 @@
int32_t result_buffer[64];
const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t);
int32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && !beta0) {
+ if (use_result_buffer && append) {
for(int cy=0; cy<std::min(M-y, 4); cy++) {
for(unsigned int cx=0; cx<width; cx++) {
result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
@@ -74,7 +76,6 @@
switch(M-y) {
case 1:
__asm __volatile (
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -89,196 +90,174 @@
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -287,37 +266,37 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
"str q19, [%[c_ptr0], #0x30]\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
);
break;
@@ -325,316 +304,283 @@
__asm __volatile (
"a_ptr1 .req X0\n"
"c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
"movi v18.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
"movi v19.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
"movi v20.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"movi v21.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
"movi v22.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
"movi v23.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"ldr q1, [a_ptr1]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "b.ne 3b\n"
+ "cbz %[loops], 1f\n"
"2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q15, [%[b_ptr0], #-0x10]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ "ldr q8, [%[b_ptr0], #-0x80]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ "ldr q9, [%[b_ptr0], #-0x70]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ "ldr q10, [%[b_ptr0], #-0x60]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ "ldr q11, [%[b_ptr0], #-0x50]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ "ldr q12, [%[b_ptr0], #-0x40]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ "ldr q13, [%[b_ptr0], #-0x30]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ "ldr q14, [%[b_ptr0], #-0x20]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ "ldr q0, [%[a_ptr0], #-0x10]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ "ldr q15, [%[b_ptr0], #-0x10]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ "ldr q1, [a_ptr1, #-0x10]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ "ldr q8, [%[b_ptr0], #-0x80]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ "ldr q9, [%[b_ptr0], #-0x70]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ "ldr q10, [%[b_ptr0], #-0x60]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ "ldr q11, [%[b_ptr0], #-0x50]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ "ldr q12, [%[b_ptr0], #-0x40]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ "ldr q13, [%[b_ptr0], #-0x30]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ "ldr q14, [%[b_ptr0], #-0x20]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"prfm PSTL1KEEP, [c_ptr1]\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "b 5f\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
"4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -643,43 +589,43 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[0], [a_ptr1], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[1], [a_ptr1], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
"ld1 {v1.b}[2], [a_ptr1]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -692,7 +638,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
);
break;
@@ -702,421 +648,378 @@
"a_ptr2 .req X1\n"
"c_ptr1 .req X2\n"
"c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
"movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
"movi v19.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
"movi v20.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"movi v21.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
"movi v22.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
"movi v23.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
"movi v24.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"movi v25.4s, #0\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v26.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
- "ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
"ldr q1, [a_ptr1]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
+ "movi v26.4s, #0\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "movi v27.4s, #0\n"
"ldr q2, [a_ptr2]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "b.ne 3b\n"
+ "cbz %[loops], 1f\n"
"2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q15, [%[b_ptr0], #-0x10]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "ldr q6, [a_ptr2]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ "ldr q8, [%[b_ptr0], #-0x80]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ "ldr q9, [%[b_ptr0], #-0x70]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ "ldr q10, [%[b_ptr0], #-0x60]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ "ldr q11, [%[b_ptr0], #-0x50]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ "ldr q12, [%[b_ptr0], #-0x40]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ "ldr q13, [%[b_ptr0], #-0x30]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ "ldr q14, [%[b_ptr0], #-0x20]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ "ldr q0, [%[a_ptr0], #-0x10]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ "ldr q1, [a_ptr1, #-0x10]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ "ldr q15, [%[b_ptr0], #-0x10]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ "ldr q2, [a_ptr2, #-0x10]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ "ldr q8, [%[b_ptr0], #-0x80]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ "ldr q9, [%[b_ptr0], #-0x70]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ "ldr q10, [%[b_ptr0], #-0x60]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ "ldr q11, [%[b_ptr0], #-0x50]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ "ldr q12, [%[b_ptr0], #-0x40]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ "ldr q13, [%[b_ptr0], #-0x30]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ "ldr q14, [%[b_ptr0], #-0x20]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"prfm PSTL1KEEP, [c_ptr1]\n"
"prfm PSTL1KEEP, [c_ptr2]\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr q6, [a_ptr2]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "b 5f\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
"4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -1125,56 +1028,56 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[0], [a_ptr1], #1\n"
"ld1 {v2.b}[0], [a_ptr2], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[1], [a_ptr1], #1\n"
"ld1 {v2.b}[1], [a_ptr2], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
"ld1 {v1.b}[2], [a_ptr1]\n"
"ld1 {v2.b}[2], [a_ptr2]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -1193,7 +1096,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -1206,526 +1109,473 @@
"c_ptr1 .req X3\n"
"c_ptr2 .req X4\n"
"c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[beta0], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
"movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
"movi v19.4s, #0\n"
- "ldr q3, [a_ptr3]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
"movi v20.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"movi v21.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
"movi v22.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
"movi v23.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
"movi v24.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"movi v25.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v26.4s, #0\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v28.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "movi v29.4s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "movi v30.4s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "movi v31.4s, #0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
- "ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
- "ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
- "ldr q28, [c_ptr3]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "mul v28.4s, v28.4s, v15.4s\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mul v29.4s, v29.4s, v15.4s\n"
"ldr q1, [a_ptr1]\n"
- "mul v30.4s, v30.4s, v15.4s\n"
+ "movi v26.4s, #0\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "movi v27.4s, #0\n"
"ldr q2, [a_ptr2]\n"
- "mul v31.4s, v31.4s, v15.4s\n"
+ "movi v28.4s, #0\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "movi v29.4s, #0\n"
"ldr q3, [a_ptr3]\n"
- "ldr q8, [%[b_ptr0]]\n"
+ "movi v30.4s, #0\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "movi v31.4s, #0\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
"add a_ptr3, a_ptr3, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q7, [a_ptr3]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
- "b.ne 3b\n"
+ "cbz %[loops], 1f\n"
"2:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q15, [%[b_ptr0], #-0x10]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ "ldr q6, [a_ptr2]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "ldr q7, [a_ptr3]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
+ "ldr q8, [%[b_ptr0], #-0x80]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
+ "ldr q9, [%[b_ptr0], #-0x70]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
+ "ldr q10, [%[b_ptr0], #-0x60]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
+ "ldr q11, [%[b_ptr0], #-0x50]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
+ "ldr q12, [%[b_ptr0], #-0x40]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
+ "ldr q13, [%[b_ptr0], #-0x30]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
+ "ldr q14, [%[b_ptr0], #-0x20]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ "ldr q0, [%[a_ptr0], #-0x10]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ "ldr q1, [a_ptr1, #-0x10]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ "ldr q2, [a_ptr2, #-0x10]\n"
+ ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
+ "ldr q15, [%[b_ptr0], #-0x10]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ "ldr q3, [a_ptr3, #-0x10]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x100\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
+ "ldr q8, [%[b_ptr0], #-0x80]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
+ "ldr q9, [%[b_ptr0], #-0x70]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
+ "ldr q10, [%[b_ptr0], #-0x60]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
+ "ldr q11, [%[b_ptr0], #-0x50]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
+ "ldr q12, [%[b_ptr0], #-0x40]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
+ "ldr q13, [%[b_ptr0], #-0x30]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
+ "ldr q14, [%[b_ptr0], #-0x20]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"prfm PSTL1KEEP, [c_ptr1]\n"
"prfm PSTL1KEEP, [c_ptr2]\n"
"prfm PSTL1KEEP, [c_ptr3]\n"
- "cbz %[regs], 4f\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "cbz %[regs], 3f\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"ldr q6, [a_ptr2]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
"ldr q7, [a_ptr3]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
"add a_ptr3, a_ptr3, #0x10\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
+ ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
+ ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
+ ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
+ ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
+ ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
+ ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
+ ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
+ ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
- ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
- ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
- ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
- ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
- ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
- ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
- ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
- "b 5f\n"
+ ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
+ ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
+ ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
+ ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
+ ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
+ ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
+ ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
+ ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
+ "ldr q12, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
+ "ldr q13, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
+ "ldr q14, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
+ "ldr q15, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
"4:\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
"ldr q8, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
@@ -1734,69 +1584,69 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
"ldr s3, [a_ptr3]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
"add a_ptr3, a_ptr3, #0x4\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
"ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[0], [a_ptr1], #1\n"
"ld1 {v2.b}[0], [a_ptr2], #1\n"
"ld1 {v3.b}[0], [a_ptr3], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
"ld1 {v1.b}[1], [a_ptr1], #1\n"
"ld1 {v2.b}[1], [a_ptr2], #1\n"
"ld1 {v3.b}[1], [a_ptr3], #1\n"
"subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
+ "b.eq 8f\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
"ld1 {v1.b}[2], [a_ptr1]\n"
"ld1 {v2.b}[2], [a_ptr2]\n"
"ld1 {v3.b}[2], [a_ptr3]\n"
- "9:\n"
+ "8:\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "8:\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
+ "7:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
"str q18, [%[c_ptr0], #0x20]\n"
@@ -1821,7 +1671,7 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp
index 48731ef..5295650 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,8 +32,8 @@
{
// Actual kernel implementations
-void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
-void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
class hybrid_u8u32_dot_16x4
{
@@ -41,7 +41,7 @@
typedef uint8_t operand_type;
typedef uint32_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+ typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -54,11 +54,26 @@
return 16;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {};
// Default to the generic kernel
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp
index e343d58..e8ed0c3 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,14 +25,17 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0u);
+void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) {
+ UNUSED(bias);
+ UNUSED(act);
+
const int K_stride = ((K + 3) / 4) * 4;
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
@@ -49,7 +52,6 @@
for (int x0=0; x0<N; x0+=16ul) {
const long width = std::min((unsigned long)N-x0, 16ul);
- const uint32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long blocks = blocks_count;
@@ -60,7 +62,7 @@
uint32_t result_buffer[64];
const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t);
uint32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && !beta0) {
+ if (use_result_buffer && append) {
for(int cy=0; cy<std::min(M-y, 4); cy++) {
for(unsigned int cx=0; cx<width; cx++) {
result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
@@ -78,7 +80,7 @@
"temploadreg1 .req X1\n"
"temploadreg2 .req X2\n"
"temploadreg3 .req X3\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -97,21 +99,16 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -119,19 +116,19 @@
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -146,108 +143,108 @@
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
"ldr d8, [%[b_ptr0]]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ins v0.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
@@ -266,21 +263,21 @@
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"ins v15.d[1], temploadreg3\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
@@ -291,134 +288,134 @@
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
"ins v13.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"ldr d8, [%[b_ptr0]]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
"ins v12.d[1], temploadreg0\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -430,10 +427,10 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -449,10 +446,10 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -464,7 +461,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -478,7 +475,7 @@
"temploadreg3 .req X5\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -503,31 +500,22 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -535,33 +523,33 @@
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -571,83 +559,83 @@
"add %[a_ptr0], %[a_ptr0], #0x20\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
"ldr d1, [a_ptr1, #-0x10]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
"ldr d8, [%[b_ptr0]]\n"
"ins v0.d[1], temploadreg0\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ins v1.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
@@ -656,55 +644,55 @@
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
@@ -723,211 +711,211 @@
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
"ins v15.d[1], temploadreg3\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -939,16 +927,16 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -967,14 +955,14 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -992,7 +980,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
@@ -1010,7 +998,7 @@
"add c_ptr1, %[c_ptr0], %[ldc]\n"
"add a_ptr2, a_ptr1, %[lda]\n"
"add c_ptr2, c_ptr1, %[ldc]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -1042,41 +1030,28 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q2, [a_ptr2]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -1085,149 +1060,149 @@
"ins v14.d[1], temploadreg2\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
"ins v10.d[1], temploadreg2\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
"ldr d1, [a_ptr1, #-0x10]\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ins v0.d[1], temploadreg0\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ins v1.d[1], temploadreg1\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
"add a_ptr2, a_ptr2, #0x20\n"
@@ -1235,65 +1210,65 @@
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
"ldr d2, [a_ptr2, #-0x10]\n"
"prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
"ins v8.d[1], temploadreg0\n"
"ins v2.d[1], temploadreg2\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
"ins v9.d[1], temploadreg1\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
@@ -1313,263 +1288,263 @@
"prfm PSTL1KEEP, [c_ptr2]\n"
"ins v15.d[1], temploadreg3\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -1581,22 +1556,22 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -1618,18 +1593,18 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -1653,7 +1628,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
);
break;
@@ -1676,7 +1651,7 @@
"add c_ptr2, c_ptr1, %[ldc]\n"
"add a_ptr3, a_ptr2, %[lda]\n"
"add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -1714,51 +1689,34 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
"ldr q28, [c_ptr3]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
"ldr q29, [c_ptr3, #0x10]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
"ldr q30, [c_ptr3, #0x20]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
"ldr q31, [c_ptr3, #0x30]\n"
- "mul v28.4s, v28.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v29.4s, v29.4s, v15.4s\n"
- "ldr q1, [a_ptr1]\n"
- "mul v30.4s, v30.4s, v15.4s\n"
- "ldr q2, [a_ptr2]\n"
- "mul v31.4s, v31.4s, v15.4s\n"
- "ldr q3, [a_ptr3]\n"
- "ldr q8, [%[b_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q1, [a_ptr1]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q2, [a_ptr2]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q3, [a_ptr3]\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
@@ -1767,261 +1725,261 @@
"ins v14.d[1], temploadreg2\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d7, [a_ptr3]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ins v7.d[1], temploadreg3\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
"ldr d0, [%[a_ptr0], #-0x10]\n"
- ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
+ ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
"ins v0.d[1], temploadreg0\n"
- ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
+ ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
"ldr d1, [a_ptr1, #-0x10]\n"
- ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
+ ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
"ins v1.d[1], temploadreg1\n"
- ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
+ ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
"add a_ptr2, a_ptr2, #0x20\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
"ldr d2, [a_ptr2, #-0x10]\n"
- ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
"ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
"ins v2.d[1], temploadreg2\n"
- ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
"add a_ptr3, a_ptr3, #0x20\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
"ldr d3, [a_ptr3, #-0x10]\n"
- ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
+ ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
"ldr temploadreg3, [a_ptr3, #-0x8]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
"ins v3.d[1], temploadreg3\n"
- ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
+ ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
+ ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
+ ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
+ ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
+ ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
+ ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
+ ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
"ins v12.d[1], temploadreg0\n"
"ins v13.d[1], temploadreg1\n"
"prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
@@ -2036,315 +1994,315 @@
"prfm PSTL1KEEP, [c_ptr3]\n"
"ins v15.d[1], temploadreg3\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr d4, [%[a_ptr0]]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr d5, [a_ptr1]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
"ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr d6, [a_ptr2]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr d7, [a_ptr3]\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
"ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ins v4.d[1], temploadreg0\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ins v5.d[1], temploadreg1\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"ins v6.d[1], temploadreg2\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ins v7.d[1], temploadreg3\n"
- ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr d8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
"ldr d9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
"ldr d10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
"ldr d11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
"ldr d12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
"ldr d13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
"ldr d14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
"ldr d15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
+ ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"add a_ptr3, a_ptr3, #0x10\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
+ ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
+ ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
+ ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
"ldr d8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ins v8.d[1], temploadreg0\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
"ldr d9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ins v9.d[1], temploadreg1\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
"ldr d10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ins v10.d[1], temploadreg2\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"ldr d11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ins v11.d[1], temploadreg3\n"
- ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
"ldr d12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ins v12.d[1], temploadreg0\n"
- ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
"ldr d13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ins v13.d[1], temploadreg1\n"
- ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
"ldr d14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ins v14.d[1], temploadreg2\n"
- ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
"ldr d15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ins v15.d[1], temploadreg3\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -2356,28 +2314,28 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr s3, [a_ptr3]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"add a_ptr3, a_ptr3, #0x4\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -2402,22 +2360,22 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -2447,7 +2405,7 @@
".unreq temploadreg2\n"
".unreq temploadreg3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
);
break;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp
index 6301fa5..23d919a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,14 +25,17 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0u);
+void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) {
+ UNUSED(bias);
+ UNUSED(act);
+
const int K_stride = ((K + 3) / 4) * 4;
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
@@ -49,7 +52,6 @@
for (int x0=0; x0<N; x0+=16ul) {
const long width = std::min((unsigned long)N-x0, 16ul);
- const uint32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long blocks = blocks_count;
@@ -60,7 +62,7 @@
uint32_t result_buffer[64];
const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t);
uint32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && !beta0) {
+ if (use_result_buffer && append) {
for(int cy=0; cy<std::min(M-y, 4); cy++) {
for(unsigned int cx=0; cx<width; cx++) {
result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
@@ -74,7 +76,7 @@
switch(M-y) {
case 1:
__asm __volatile (
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -92,93 +94,88 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
@@ -189,93 +186,93 @@
"ldr q15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -287,10 +284,10 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -306,10 +303,10 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -317,7 +314,7 @@
"str q19, [%[c_ptr0], #0x30]\n"
"add %[c_ptr0], %[c_ptr0], #0x40\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
);
break;
@@ -327,7 +324,7 @@
"c_ptr1 .req X1\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -351,287 +348,278 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr q1, [a_ptr1, #-0x10]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
"b.ne 3b\n"
"2:\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
"prfm PSTL1KEEP, [%[c_ptr0]]\n"
"prfm PSTL1KEEP, [c_ptr1]\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -643,16 +631,16 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -671,14 +659,14 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -692,7 +680,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
);
break;
@@ -706,7 +694,7 @@
"add c_ptr1, %[c_ptr0], %[ldc]\n"
"add a_ptr2, a_ptr1, %[lda]\n"
"add c_ptr2, c_ptr1, %[ldc]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -736,190 +724,177 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q1, [a_ptr1]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ldr q2, [a_ptr2]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ldr q8, [%[b_ptr0]]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q6, [a_ptr2]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"add a_ptr2, a_ptr2, #0x20\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ldr q1, [a_ptr1, #-0x10]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr q2, [a_ptr2, #-0x10]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
"b.ne 3b\n"
"2:\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
@@ -927,193 +902,193 @@
"prfm PSTL1KEEP, [c_ptr1]\n"
"prfm PSTL1KEEP, [c_ptr2]\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr q6, [a_ptr2]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -1125,22 +1100,22 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -1162,18 +1137,18 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -1193,7 +1168,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -1212,7 +1187,7 @@
"add c_ptr2, c_ptr1, %[ldc]\n"
"add a_ptr3, a_ptr2, %[lda]\n"
"add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"movi v16.4s, #0\n"
"ldr q0, [%[a_ptr0]]\n"
"movi v17.4s, #0\n"
@@ -1248,236 +1223,219 @@
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1r {v15.4s}, [%[betaptr]]\n"
"ldr q16, [%[c_ptr0]]\n"
"ldr q17, [%[c_ptr0], #0x10]\n"
"ldr q18, [%[c_ptr0], #0x20]\n"
"ldr q19, [%[c_ptr0], #0x30]\n"
- "mul v16.4s, v16.4s, v15.4s\n"
"ldr q20, [c_ptr1]\n"
- "mul v17.4s, v17.4s, v15.4s\n"
"ldr q21, [c_ptr1, #0x10]\n"
- "mul v18.4s, v18.4s, v15.4s\n"
"ldr q22, [c_ptr1, #0x20]\n"
- "mul v19.4s, v19.4s, v15.4s\n"
"ldr q23, [c_ptr1, #0x30]\n"
- "mul v20.4s, v20.4s, v15.4s\n"
"ldr q24, [c_ptr2]\n"
- "mul v21.4s, v21.4s, v15.4s\n"
"ldr q25, [c_ptr2, #0x10]\n"
- "mul v22.4s, v22.4s, v15.4s\n"
"ldr q26, [c_ptr2, #0x20]\n"
- "mul v23.4s, v23.4s, v15.4s\n"
"ldr q27, [c_ptr2, #0x30]\n"
- "mul v24.4s, v24.4s, v15.4s\n"
"ldr q28, [c_ptr3]\n"
- "mul v25.4s, v25.4s, v15.4s\n"
"ldr q29, [c_ptr3, #0x10]\n"
- "mul v26.4s, v26.4s, v15.4s\n"
"ldr q30, [c_ptr3, #0x20]\n"
- "mul v27.4s, v27.4s, v15.4s\n"
"ldr q31, [c_ptr3, #0x30]\n"
- "mul v28.4s, v28.4s, v15.4s\n"
"ldr q0, [%[a_ptr0]]\n"
- "mul v29.4s, v29.4s, v15.4s\n"
- "ldr q1, [a_ptr1]\n"
- "mul v30.4s, v30.4s, v15.4s\n"
- "ldr q2, [a_ptr2]\n"
- "mul v31.4s, v31.4s, v15.4s\n"
- "ldr q3, [a_ptr3]\n"
- "ldr q8, [%[b_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q1, [a_ptr1]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q2, [a_ptr2]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q3, [a_ptr3]\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"cbz %[loops], 2f\n"
"3:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
"ldr q6, [a_ptr2]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q7, [a_ptr3]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"subs %[loops], %[loops], #0x1\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
"add a_ptr1, a_ptr1, #0x20\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
"add a_ptr2, a_ptr2, #0x20\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
"add a_ptr3, a_ptr3, #0x20\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
"ldr q0, [%[a_ptr0], #-0x10]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
"ldr q1, [a_ptr1, #-0x10]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
"ldr q2, [a_ptr2, #-0x10]\n"
- ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
"ldr q3, [a_ptr3, #-0x10]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
"b.ne 3b\n"
"2:\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
@@ -1486,243 +1444,243 @@
"prfm PSTL1KEEP, [c_ptr2]\n"
"prfm PSTL1KEEP, [c_ptr3]\n"
"cbz %[regs], 4f\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr q4, [%[a_ptr0]]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"ldr q5, [a_ptr1]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"ldr q6, [a_ptr2]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
"ldr q7, [a_ptr3]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
"add a_ptr3, a_ptr3, #0x10\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
"ldr q8, [%[b_ptr0], #-0x80]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
"ldr q9, [%[b_ptr0], #-0x70]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
"ldr q10, [%[b_ptr0], #-0x60]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
"ldr q11, [%[b_ptr0], #-0x50]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
"ldr q12, [%[b_ptr0], #-0x40]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
"ldr q13, [%[b_ptr0], #-0x30]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
"ldr q14, [%[b_ptr0], #-0x20]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
"ldr q15, [%[b_ptr0], #-0x10]\n"
- ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
+ ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
+ ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
+ ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
+ ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
+ ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
+ ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
+ ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
+ ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
+ ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
+ ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
+ ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
+ ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
+ ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
+ ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
+ ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
+ ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
+ ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
+ ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
- ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
- ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
- ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
- ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
- ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
- ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
- ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
+ ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
+ ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
+ ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
+ ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
+ ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
+ ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
+ ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
+ ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
+ ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
+ ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
+ ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
+ ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
+ ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
+ ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
+ ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
+ ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
+ ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
+ ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
+ ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
"b 5f\n"
"4:\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
"ldr q8, [%[b_ptr0]]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
"ldr q9, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
"ldr q12, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
"ldr q13, [%[b_ptr0], #0x50]\n"
- ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
"ldr q14, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
"ldr q15, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
+ ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
+ ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
+ ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
+ ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
+ ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
"5:\n"
"cbz %[blocks], 6f\n"
"7:\n"
@@ -1734,28 +1692,28 @@
"add %[a_ptr0], %[a_ptr0], #0x4\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
"ldr s1, [a_ptr1]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
"add a_ptr1, a_ptr1, #0x4\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
"ldr s2, [a_ptr2]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
"add a_ptr2, a_ptr2, #0x4\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
"ldr s3, [a_ptr3]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
"add a_ptr3, a_ptr3, #0x4\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"b.ne 7b\n"
"6:\n"
"cbz %[odds], 8f\n"
@@ -1780,22 +1738,22 @@
"ldr q9, [%[b_ptr0], #0x10]\n"
"ldr q10, [%[b_ptr0], #0x20]\n"
"ldr q11, [%[b_ptr0], #0x30]\n"
- ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
+ ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
+ ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
+ ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
"8:\n"
"str q16, [%[c_ptr0]]\n"
"str q17, [%[c_ptr0], #0x10]\n"
@@ -1821,7 +1779,7 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb)
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp
similarity index 67%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp
index 06622d6..a86e8ec 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,51 +23,61 @@
*/
#pragma once
-#ifdef __ARM_FEATURE_SVE
-
-
+#ifdef __aarch64__
namespace arm_gemm
{
// Actual kernel implementations
-void sve_smallK_fp32_mla_1VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+void a64_native_fp32_mla_16x4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool);
-class smallK_fp32_mla_1VLx4
+class native_fp32_mla_16x4
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+ typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
static unsigned int out_width()
{
- return get_vector_length<float>() * 1;
+ return 16;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
// Default to the generic kernel
- kern_type kernel=sve_smallK_fp32_mla_1VLx4;
+ kern_type kernel=a64_native_fp32_mla_16x4;
- smallK_fp32_mla_1VLx4(const CPUInfo *ci)
- {
-
- }
+ native_fp32_mla_16x4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
-#endif // __ARM_FEATURE_SVE
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp
new file mode 100644
index 0000000..82e7333
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp
@@ -0,0 +1,1708 @@
+/*
+ * Copyright (c) 2018-2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include <algorithm>
+
+#include "arm_gemm.hpp"
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void a64_native_fp32_mla_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) {
+ const long loops_count = ((K + 4) / 8) - 1;
+ K -= loops_count * 8;
+ const long regs_count = (K / 4) - 1;
+ K -= (regs_count + 1) * 4;
+ const long blocks_count = K / 1;
+ float nullbias[16];
+ if (!append && !bias) {
+ memset(nullbias, 0, (16 * sizeof(float)));
+ }
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ const float * const minptr = &minval;
+ const float * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ for (int y=0; y<M; y+=4) {
+ const float * const a_ptr0_base = A + (y * lda);
+ const unsigned long ldab = lda * sizeof(float);
+
+ float *c_ptr0 = C + (y * ldc);
+
+ for (int x0=0; x0<N; x0+=16ul) {
+ const long width = std::min((unsigned long)N-x0, 16ul);
+ long loops = loops_count;
+ long regs = regs_count;
+ long blocks = blocks_count;
+ const float *a_ptr0 = a_ptr0_base;
+ const float *b_ptr0 = B + x0;
+ long ldbb = ldb * sizeof(float);
+ const unsigned long ldcb = ldc * sizeof(float);
+ const float *biasptr = bias ? bias+x0 : nullbias;
+
+ switch(M-y) {
+ case 1:
+ __asm __volatile (
+ "ldr q16, [%[biasptr]]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "ldr q0, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "ldr q0, [%[a_ptr0], #-0x10]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0]]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "cbz %[regs], 3f\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "b 4f\n"
+ "3:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr s0, [%[a_ptr0]]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x4\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
+ "str q16, [%[c_ptr0]]\n"
+ "str q17, [%[c_ptr0], #0x10]\n"
+ "str q18, [%[c_ptr0], #0x20]\n"
+ "str q19, [%[c_ptr0], #0x30]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x40\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "c_ptr1 .req X1\n"
+ "ldr q16, [%[biasptr]]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "mov v20.16b, v16.16b\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v21.16b, v17.16b\n"
+ "ldr q0, [%[a_ptr0]]\n"
+ "ldr q1, [a_ptr1]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov v22.16b, v18.16b\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "mov v23.16b, v19.16b\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "ldr q0, [%[a_ptr0], #-0x10]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "ldr q1, [a_ptr1, #-0x10]\n"
+ "fmla v20.4s, v8.4s, v5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v21.4s, v9.4s, v5.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "fmla v22.4s, v10.4s, v5.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "fmla v23.4s, v11.4s, v5.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "fmla v21.4s, v13.4s, v5.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "fmla v22.4s, v14.4s, v5.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v5.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "fmla v21.4s, v9.4s, v5.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "fmla v23.4s, v11.4s, v5.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "fmla v21.4s, v13.4s, v5.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "fmla v22.4s, v14.4s, v5.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "fmla v23.4s, v15.4s, v5.s[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0]]\n"
+ "prfm PSTL1KEEP, [c_ptr1]\n"
+ "cbz %[regs], 3f\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v5.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "fmla v21.4s, v9.4s, v5.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "fmla v22.4s, v10.4s, v5.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "fmla v23.4s, v11.4s, v5.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "fmla v21.4s, v13.4s, v5.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "fmla v22.4s, v14.4s, v5.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "fmla v20.4s, v8.4s, v5.s[2]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "fmla v21.4s, v9.4s, v5.s[2]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "fmla v23.4s, v11.4s, v5.s[2]\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "fmla v20.4s, v12.4s, v5.s[3]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "fmla v21.4s, v13.4s, v5.s[3]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "fmla v22.4s, v14.4s, v5.s[3]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "fmla v23.4s, v15.4s, v5.s[3]\n"
+ "b 4f\n"
+ "3:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr s0, [%[a_ptr0]]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x4\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr s1, [a_ptr1]\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add a_ptr1, a_ptr1, #0x4\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
+ "str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
+ "str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
+ "str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
+ "str q19, [%[c_ptr0], #0x30]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x40\n"
+ "str q20, [c_ptr1]\n"
+ "str q21, [c_ptr1, #0x10]\n"
+ "str q22, [c_ptr1, #0x20]\n"
+ "str q23, [c_ptr1, #0x30]\n"
+ ".unreq a_ptr1\n"
+ ".unreq c_ptr1\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "c_ptr1 .req X2\n"
+ "c_ptr2 .req X3\n"
+ "ldr q16, [%[biasptr]]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "mov v20.16b, v16.16b\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "mov v24.16b, v16.16b\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v21.16b, v17.16b\n"
+ "ldr q0, [%[a_ptr0]]\n"
+ "mov v25.16b, v17.16b\n"
+ "ldr q1, [a_ptr1]\n"
+ "mov v22.16b, v18.16b\n"
+ "ldr q2, [a_ptr2]\n"
+ "mov v23.16b, v19.16b\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "mov v26.16b, v18.16b\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "mov v27.16b, v19.16b\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "ldr q6, [a_ptr2]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla v24.4s, v12.4s, v2.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ "fmla v25.4s, v13.4s, v2.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "fmla v26.4s, v14.4s, v2.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "fmla v27.4s, v15.4s, v2.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "fmla v24.4s, v8.4s, v2.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "fmla v25.4s, v9.4s, v2.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "fmla v27.4s, v11.4s, v2.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v24.4s, v12.4s, v2.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v25.4s, v13.4s, v2.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v26.4s, v14.4s, v2.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "ldr q0, [%[a_ptr0], #-0x10]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "ldr q1, [a_ptr1, #-0x10]\n"
+ "fmla v27.4s, v15.4s, v2.s[3]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "ldr q2, [a_ptr2, #-0x10]\n"
+ "fmla v20.4s, v8.4s, v5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v24.4s, v8.4s, v6.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "fmla v21.4s, v9.4s, v5.s[0]\n"
+ "fmla v25.4s, v9.4s, v6.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "fmla v22.4s, v10.4s, v5.s[0]\n"
+ "fmla v26.4s, v10.4s, v6.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "fmla v23.4s, v11.4s, v5.s[0]\n"
+ "fmla v27.4s, v11.4s, v6.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[1]\n"
+ "fmla v24.4s, v12.4s, v6.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "fmla v21.4s, v13.4s, v5.s[1]\n"
+ "fmla v25.4s, v13.4s, v6.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "fmla v22.4s, v14.4s, v5.s[1]\n"
+ "fmla v26.4s, v14.4s, v6.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "fmla v27.4s, v15.4s, v6.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v5.s[2]\n"
+ "fmla v24.4s, v8.4s, v6.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "fmla v21.4s, v9.4s, v5.s[2]\n"
+ "fmla v25.4s, v9.4s, v6.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "fmla v26.4s, v10.4s, v6.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "fmla v23.4s, v11.4s, v5.s[2]\n"
+ "fmla v27.4s, v11.4s, v6.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[3]\n"
+ "fmla v24.4s, v12.4s, v6.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "fmla v21.4s, v13.4s, v5.s[3]\n"
+ "fmla v25.4s, v13.4s, v6.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "fmla v22.4s, v14.4s, v5.s[3]\n"
+ "fmla v26.4s, v14.4s, v6.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "fmla v23.4s, v15.4s, v5.s[3]\n"
+ "fmla v27.4s, v15.4s, v6.s[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0]]\n"
+ "prfm PSTL1KEEP, [c_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr2]\n"
+ "cbz %[regs], 3f\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "ldr q6, [a_ptr2]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "fmla v24.4s, v12.4s, v2.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "fmla v25.4s, v13.4s, v2.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "fmla v26.4s, v14.4s, v2.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "fmla v27.4s, v15.4s, v2.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "fmla v24.4s, v8.4s, v2.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "fmla v25.4s, v9.4s, v2.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "fmla v27.4s, v11.4s, v2.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v24.4s, v12.4s, v2.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v25.4s, v13.4s, v2.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v26.4s, v14.4s, v2.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "fmla v27.4s, v15.4s, v2.s[3]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v5.s[0]\n"
+ "fmla v24.4s, v8.4s, v6.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "fmla v21.4s, v9.4s, v5.s[0]\n"
+ "fmla v25.4s, v9.4s, v6.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "fmla v22.4s, v10.4s, v5.s[0]\n"
+ "fmla v26.4s, v10.4s, v6.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "fmla v23.4s, v11.4s, v5.s[0]\n"
+ "fmla v27.4s, v11.4s, v6.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[1]\n"
+ "fmla v24.4s, v12.4s, v6.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "fmla v21.4s, v13.4s, v5.s[1]\n"
+ "fmla v25.4s, v13.4s, v6.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "fmla v22.4s, v14.4s, v5.s[1]\n"
+ "fmla v26.4s, v14.4s, v6.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "fmla v27.4s, v15.4s, v6.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "fmla v20.4s, v8.4s, v5.s[2]\n"
+ "fmla v24.4s, v8.4s, v6.s[2]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "fmla v21.4s, v9.4s, v5.s[2]\n"
+ "fmla v25.4s, v9.4s, v6.s[2]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "fmla v26.4s, v10.4s, v6.s[2]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "fmla v23.4s, v11.4s, v5.s[2]\n"
+ "fmla v27.4s, v11.4s, v6.s[2]\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "fmla v20.4s, v12.4s, v5.s[3]\n"
+ "fmla v24.4s, v12.4s, v6.s[3]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "fmla v21.4s, v13.4s, v5.s[3]\n"
+ "fmla v25.4s, v13.4s, v6.s[3]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "fmla v22.4s, v14.4s, v5.s[3]\n"
+ "fmla v26.4s, v14.4s, v6.s[3]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "fmla v23.4s, v15.4s, v5.s[3]\n"
+ "fmla v27.4s, v15.4s, v6.s[3]\n"
+ "b 4f\n"
+ "3:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "fmla v24.4s, v12.4s, v2.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "fmla v25.4s, v13.4s, v2.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "fmla v26.4s, v14.4s, v2.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "fmla v27.4s, v15.4s, v2.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "fmla v24.4s, v8.4s, v2.s[2]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "fmla v25.4s, v9.4s, v2.s[2]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "fmla v27.4s, v11.4s, v2.s[2]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v24.4s, v12.4s, v2.s[3]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v25.4s, v13.4s, v2.s[3]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v26.4s, v14.4s, v2.s[3]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "fmla v27.4s, v15.4s, v2.s[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr s0, [%[a_ptr0]]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x4\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr s1, [a_ptr1]\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add a_ptr1, a_ptr1, #0x4\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr s2, [a_ptr2]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x4\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
+ "str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
+ "str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
+ "str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
+ "fmax v24.4s, v24.4s, v14.4s\n"
+ "str q19, [%[c_ptr0], #0x30]\n"
+ "fmax v25.4s, v25.4s, v14.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x40\n"
+ "fmax v26.4s, v26.4s, v14.4s\n"
+ "str q20, [c_ptr1]\n"
+ "fmin v24.4s, v24.4s, v15.4s\n"
+ "fmin v25.4s, v25.4s, v15.4s\n"
+ "fmax v27.4s, v27.4s, v14.4s\n"
+ "str q21, [c_ptr1, #0x10]\n"
+ "fmin v26.4s, v26.4s, v15.4s\n"
+ "fmin v27.4s, v27.4s, v15.4s\n"
+ "str q22, [c_ptr1, #0x20]\n"
+ "str q23, [c_ptr1, #0x30]\n"
+ "str q24, [c_ptr2]\n"
+ "str q25, [c_ptr2, #0x10]\n"
+ "str q26, [c_ptr2, #0x20]\n"
+ "str q27, [c_ptr2, #0x30]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
+ );
+ break;
+ default:
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "ldr q16, [%[biasptr]]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "ldr q17, [%[biasptr], #0x10]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "mov v20.16b, v16.16b\n"
+ "ldr q18, [%[biasptr], #0x20]\n"
+ "mov v24.16b, v16.16b\n"
+ "ldr q19, [%[biasptr], #0x30]\n"
+ "mov v21.16b, v17.16b\n"
+ "ldr q0, [%[a_ptr0]]\n"
+ "mov v25.16b, v17.16b\n"
+ "ldr q1, [a_ptr1]\n"
+ "mov v22.16b, v18.16b\n"
+ "ldr q2, [a_ptr2]\n"
+ "mov v23.16b, v19.16b\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "mov v26.16b, v18.16b\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "mov v27.16b, v19.16b\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "mov v28.16b, v16.16b\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "mov v29.16b, v17.16b\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "mov v30.16b, v18.16b\n"
+ "ldr q3, [a_ptr3]\n"
+ "mov v31.16b, v19.16b\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ "fmla v28.4s, v8.4s, v3.s[0]\n"
+ "ldr q6, [a_ptr2]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "ldr q7, [a_ptr3]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v29.4s, v9.4s, v3.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla v30.4s, v10.4s, v3.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
+ "fmla v31.4s, v11.4s, v3.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
+ "fmla v24.4s, v12.4s, v2.s[1]\n"
+ "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
+ "fmla v28.4s, v12.4s, v3.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "fmla v25.4s, v13.4s, v2.s[1]\n"
+ "fmla v29.4s, v13.4s, v3.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "fmla v26.4s, v14.4s, v2.s[1]\n"
+ "fmla v30.4s, v14.4s, v3.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "fmla v27.4s, v15.4s, v2.s[1]\n"
+ "fmla v31.4s, v15.4s, v3.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "fmla v24.4s, v8.4s, v2.s[2]\n"
+ "fmla v28.4s, v8.4s, v3.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "fmla v25.4s, v9.4s, v2.s[2]\n"
+ "fmla v29.4s, v9.4s, v3.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v30.4s, v10.4s, v3.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "fmla v27.4s, v11.4s, v2.s[2]\n"
+ "fmla v31.4s, v11.4s, v3.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v24.4s, v12.4s, v2.s[3]\n"
+ "fmla v28.4s, v12.4s, v3.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v25.4s, v13.4s, v2.s[3]\n"
+ "fmla v29.4s, v13.4s, v3.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v26.4s, v14.4s, v2.s[3]\n"
+ "fmla v30.4s, v14.4s, v3.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "ldr q0, [%[a_ptr0], #-0x10]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "ldr q1, [a_ptr1, #-0x10]\n"
+ "fmla v27.4s, v15.4s, v2.s[3]\n"
+ "ldr q2, [a_ptr2, #-0x10]\n"
+ "fmla v31.4s, v15.4s, v3.s[3]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "ldr q3, [a_ptr3, #-0x10]\n"
+ "fmla v20.4s, v8.4s, v5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v24.4s, v8.4s, v6.s[0]\n"
+ "fmla v28.4s, v8.4s, v7.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "fmla v21.4s, v9.4s, v5.s[0]\n"
+ "fmla v25.4s, v9.4s, v6.s[0]\n"
+ "fmla v29.4s, v9.4s, v7.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "fmla v22.4s, v10.4s, v5.s[0]\n"
+ "fmla v26.4s, v10.4s, v6.s[0]\n"
+ "fmla v30.4s, v10.4s, v7.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "fmla v23.4s, v11.4s, v5.s[0]\n"
+ "fmla v27.4s, v11.4s, v6.s[0]\n"
+ "fmla v31.4s, v11.4s, v7.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[1]\n"
+ "fmla v24.4s, v12.4s, v6.s[1]\n"
+ "fmla v28.4s, v12.4s, v7.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "fmla v21.4s, v13.4s, v5.s[1]\n"
+ "fmla v25.4s, v13.4s, v6.s[1]\n"
+ "fmla v29.4s, v13.4s, v7.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "fmla v22.4s, v14.4s, v5.s[1]\n"
+ "fmla v26.4s, v14.4s, v6.s[1]\n"
+ "fmla v30.4s, v14.4s, v7.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "fmla v27.4s, v15.4s, v6.s[1]\n"
+ "fmla v31.4s, v15.4s, v7.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v5.s[2]\n"
+ "fmla v24.4s, v8.4s, v6.s[2]\n"
+ "fmla v28.4s, v8.4s, v7.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "fmla v21.4s, v9.4s, v5.s[2]\n"
+ "fmla v25.4s, v9.4s, v6.s[2]\n"
+ "fmla v29.4s, v9.4s, v7.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "fmla v26.4s, v10.4s, v6.s[2]\n"
+ "fmla v30.4s, v10.4s, v7.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "fmla v23.4s, v11.4s, v5.s[2]\n"
+ "fmla v27.4s, v11.4s, v6.s[2]\n"
+ "fmla v31.4s, v11.4s, v7.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[3]\n"
+ "fmla v24.4s, v12.4s, v6.s[3]\n"
+ "fmla v28.4s, v12.4s, v7.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "fmla v21.4s, v13.4s, v5.s[3]\n"
+ "fmla v25.4s, v13.4s, v6.s[3]\n"
+ "fmla v29.4s, v13.4s, v7.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "fmla v22.4s, v14.4s, v5.s[3]\n"
+ "fmla v26.4s, v14.4s, v6.s[3]\n"
+ "fmla v30.4s, v14.4s, v7.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "fmla v23.4s, v15.4s, v5.s[3]\n"
+ "fmla v27.4s, v15.4s, v6.s[3]\n"
+ "fmla v31.4s, v15.4s, v7.s[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0]]\n"
+ "prfm PSTL1KEEP, [c_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr2]\n"
+ "prfm PSTL1KEEP, [c_ptr3]\n"
+ "cbz %[regs], 3f\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q4, [%[a_ptr0]]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr q5, [a_ptr1]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "ldr q6, [a_ptr2]\n"
+ "fmla v28.4s, v8.4s, v3.s[0]\n"
+ "ldr q7, [a_ptr3]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "fmla v29.4s, v9.4s, v3.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "fmla v30.4s, v10.4s, v3.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "fmla v31.4s, v11.4s, v3.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "fmla v24.4s, v12.4s, v2.s[1]\n"
+ "fmla v28.4s, v12.4s, v3.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "fmla v25.4s, v13.4s, v2.s[1]\n"
+ "fmla v29.4s, v13.4s, v3.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "fmla v26.4s, v14.4s, v2.s[1]\n"
+ "fmla v30.4s, v14.4s, v3.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "fmla v27.4s, v15.4s, v2.s[1]\n"
+ "fmla v31.4s, v15.4s, v3.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "fmla v24.4s, v8.4s, v2.s[2]\n"
+ "fmla v28.4s, v8.4s, v3.s[2]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "fmla v25.4s, v9.4s, v2.s[2]\n"
+ "fmla v29.4s, v9.4s, v3.s[2]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v30.4s, v10.4s, v3.s[2]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "fmla v27.4s, v11.4s, v2.s[2]\n"
+ "fmla v31.4s, v11.4s, v3.s[2]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v24.4s, v12.4s, v2.s[3]\n"
+ "fmla v28.4s, v12.4s, v3.s[3]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v25.4s, v13.4s, v2.s[3]\n"
+ "fmla v29.4s, v13.4s, v3.s[3]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v26.4s, v14.4s, v2.s[3]\n"
+ "fmla v30.4s, v14.4s, v3.s[3]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "fmla v27.4s, v15.4s, v2.s[3]\n"
+ "fmla v31.4s, v15.4s, v3.s[3]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v5.s[0]\n"
+ "fmla v24.4s, v8.4s, v6.s[0]\n"
+ "fmla v28.4s, v8.4s, v7.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v4.s[0]\n"
+ "fmla v21.4s, v9.4s, v5.s[0]\n"
+ "fmla v25.4s, v9.4s, v6.s[0]\n"
+ "fmla v29.4s, v9.4s, v7.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v4.s[0]\n"
+ "fmla v22.4s, v10.4s, v5.s[0]\n"
+ "fmla v26.4s, v10.4s, v6.s[0]\n"
+ "fmla v30.4s, v10.4s, v7.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v4.s[0]\n"
+ "fmla v23.4s, v11.4s, v5.s[0]\n"
+ "fmla v27.4s, v11.4s, v6.s[0]\n"
+ "fmla v31.4s, v11.4s, v7.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v5.s[1]\n"
+ "fmla v24.4s, v12.4s, v6.s[1]\n"
+ "fmla v28.4s, v12.4s, v7.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v4.s[1]\n"
+ "fmla v21.4s, v13.4s, v5.s[1]\n"
+ "fmla v25.4s, v13.4s, v6.s[1]\n"
+ "fmla v29.4s, v13.4s, v7.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v4.s[1]\n"
+ "fmla v22.4s, v14.4s, v5.s[1]\n"
+ "fmla v26.4s, v14.4s, v6.s[1]\n"
+ "fmla v30.4s, v14.4s, v7.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v4.s[1]\n"
+ "fmla v23.4s, v15.4s, v5.s[1]\n"
+ "fmla v27.4s, v15.4s, v6.s[1]\n"
+ "fmla v31.4s, v15.4s, v7.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v4.s[2]\n"
+ "fmla v20.4s, v8.4s, v5.s[2]\n"
+ "fmla v24.4s, v8.4s, v6.s[2]\n"
+ "fmla v28.4s, v8.4s, v7.s[2]\n"
+ "fmla v17.4s, v9.4s, v4.s[2]\n"
+ "fmla v21.4s, v9.4s, v5.s[2]\n"
+ "fmla v25.4s, v9.4s, v6.s[2]\n"
+ "fmla v29.4s, v9.4s, v7.s[2]\n"
+ "fmla v18.4s, v10.4s, v4.s[2]\n"
+ "fmla v22.4s, v10.4s, v5.s[2]\n"
+ "fmla v26.4s, v10.4s, v6.s[2]\n"
+ "fmla v30.4s, v10.4s, v7.s[2]\n"
+ "fmla v19.4s, v11.4s, v4.s[2]\n"
+ "fmla v23.4s, v11.4s, v5.s[2]\n"
+ "fmla v27.4s, v11.4s, v6.s[2]\n"
+ "fmla v31.4s, v11.4s, v7.s[2]\n"
+ "fmla v16.4s, v12.4s, v4.s[3]\n"
+ "fmla v20.4s, v12.4s, v5.s[3]\n"
+ "fmla v24.4s, v12.4s, v6.s[3]\n"
+ "fmla v28.4s, v12.4s, v7.s[3]\n"
+ "fmla v17.4s, v13.4s, v4.s[3]\n"
+ "fmla v21.4s, v13.4s, v5.s[3]\n"
+ "fmla v25.4s, v13.4s, v6.s[3]\n"
+ "fmla v29.4s, v13.4s, v7.s[3]\n"
+ "fmla v18.4s, v14.4s, v4.s[3]\n"
+ "fmla v22.4s, v14.4s, v5.s[3]\n"
+ "fmla v26.4s, v14.4s, v6.s[3]\n"
+ "fmla v30.4s, v14.4s, v7.s[3]\n"
+ "fmla v19.4s, v15.4s, v4.s[3]\n"
+ "fmla v23.4s, v15.4s, v5.s[3]\n"
+ "fmla v27.4s, v15.4s, v6.s[3]\n"
+ "fmla v31.4s, v15.4s, v7.s[3]\n"
+ "b 4f\n"
+ "3:\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "fmla v28.4s, v8.4s, v3.s[0]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "fmla v29.4s, v9.4s, v3.s[0]\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "fmla v30.4s, v10.4s, v3.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "fmla v31.4s, v11.4s, v3.s[0]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v12.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla v20.4s, v12.4s, v1.s[1]\n"
+ "fmla v24.4s, v12.4s, v2.s[1]\n"
+ "fmla v28.4s, v12.4s, v3.s[1]\n"
+ "ldr q12, [%[b_ptr0]]\n"
+ "fmla v17.4s, v13.4s, v0.s[1]\n"
+ "fmla v21.4s, v13.4s, v1.s[1]\n"
+ "fmla v25.4s, v13.4s, v2.s[1]\n"
+ "fmla v29.4s, v13.4s, v3.s[1]\n"
+ "ldr q13, [%[b_ptr0], #0x10]\n"
+ "fmla v18.4s, v14.4s, v0.s[1]\n"
+ "fmla v22.4s, v14.4s, v1.s[1]\n"
+ "fmla v26.4s, v14.4s, v2.s[1]\n"
+ "fmla v30.4s, v14.4s, v3.s[1]\n"
+ "ldr q14, [%[b_ptr0], #0x20]\n"
+ "fmla v19.4s, v15.4s, v0.s[1]\n"
+ "fmla v23.4s, v15.4s, v1.s[1]\n"
+ "fmla v27.4s, v15.4s, v2.s[1]\n"
+ "fmla v31.4s, v15.4s, v3.s[1]\n"
+ "ldr q15, [%[b_ptr0], #0x30]\n"
+ "fmla v16.4s, v8.4s, v0.s[2]\n"
+ "fmla v20.4s, v8.4s, v1.s[2]\n"
+ "fmla v24.4s, v8.4s, v2.s[2]\n"
+ "fmla v28.4s, v8.4s, v3.s[2]\n"
+ "fmla v17.4s, v9.4s, v0.s[2]\n"
+ "fmla v21.4s, v9.4s, v1.s[2]\n"
+ "fmla v25.4s, v9.4s, v2.s[2]\n"
+ "fmla v29.4s, v9.4s, v3.s[2]\n"
+ "fmla v18.4s, v10.4s, v0.s[2]\n"
+ "fmla v22.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v30.4s, v10.4s, v3.s[2]\n"
+ "fmla v19.4s, v11.4s, v0.s[2]\n"
+ "fmla v23.4s, v11.4s, v1.s[2]\n"
+ "fmla v27.4s, v11.4s, v2.s[2]\n"
+ "fmla v31.4s, v11.4s, v3.s[2]\n"
+ "fmla v16.4s, v12.4s, v0.s[3]\n"
+ "fmla v20.4s, v12.4s, v1.s[3]\n"
+ "fmla v24.4s, v12.4s, v2.s[3]\n"
+ "fmla v28.4s, v12.4s, v3.s[3]\n"
+ "fmla v17.4s, v13.4s, v0.s[3]\n"
+ "fmla v21.4s, v13.4s, v1.s[3]\n"
+ "fmla v25.4s, v13.4s, v2.s[3]\n"
+ "fmla v29.4s, v13.4s, v3.s[3]\n"
+ "fmla v18.4s, v14.4s, v0.s[3]\n"
+ "fmla v22.4s, v14.4s, v1.s[3]\n"
+ "fmla v26.4s, v14.4s, v2.s[3]\n"
+ "fmla v30.4s, v14.4s, v3.s[3]\n"
+ "fmla v19.4s, v15.4s, v0.s[3]\n"
+ "fmla v23.4s, v15.4s, v1.s[3]\n"
+ "fmla v27.4s, v15.4s, v2.s[3]\n"
+ "fmla v31.4s, v15.4s, v3.s[3]\n"
+ "4:\n"
+ "cbz %[blocks], 5f\n"
+ "6:\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ldr s0, [%[a_ptr0]]\n"
+ "ldr q8, [%[b_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x4\n"
+ "ldr q9, [%[b_ptr0], #0x10]\n"
+ "ldr s1, [a_ptr1]\n"
+ "fmla v16.4s, v8.4s, v0.s[0]\n"
+ "ldr q10, [%[b_ptr0], #0x20]\n"
+ "ldr q11, [%[b_ptr0], #0x30]\n"
+ "add a_ptr1, a_ptr1, #0x4\n"
+ "fmla v20.4s, v8.4s, v1.s[0]\n"
+ "ldr s2, [a_ptr2]\n"
+ "fmla v17.4s, v9.4s, v0.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x4\n"
+ "fmla v21.4s, v9.4s, v1.s[0]\n"
+ "ldr s3, [a_ptr3]\n"
+ "fmla v24.4s, v8.4s, v2.s[0]\n"
+ "add a_ptr3, a_ptr3, #0x4\n"
+ "fmla v25.4s, v9.4s, v2.s[0]\n"
+ "fmla v28.4s, v8.4s, v3.s[0]\n"
+ "fmla v29.4s, v9.4s, v3.s[0]\n"
+ "fmla v18.4s, v10.4s, v0.s[0]\n"
+ "fmla v22.4s, v10.4s, v1.s[0]\n"
+ "fmla v26.4s, v10.4s, v2.s[0]\n"
+ "fmla v30.4s, v10.4s, v3.s[0]\n"
+ "fmla v19.4s, v11.4s, v0.s[0]\n"
+ "fmla v23.4s, v11.4s, v1.s[0]\n"
+ "fmla v27.4s, v11.4s, v2.s[0]\n"
+ "fmla v31.4s, v11.4s, v3.s[0]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ "ld1r {v14.4s}, [%[minptr]]\n"
+ "ld1r {v15.4s}, [%[maxptr]]\n"
+ "fmax v16.4s, v16.4s, v14.4s\n"
+ "fmax v17.4s, v17.4s, v14.4s\n"
+ "fmax v18.4s, v18.4s, v14.4s\n"
+ "fmax v19.4s, v19.4s, v14.4s\n"
+ "fmin v16.4s, v16.4s, v15.4s\n"
+ "fmin v17.4s, v17.4s, v15.4s\n"
+ "fmin v18.4s, v18.4s, v15.4s\n"
+ "fmin v19.4s, v19.4s, v15.4s\n"
+ "str q16, [%[c_ptr0]]\n"
+ "fmax v20.4s, v20.4s, v14.4s\n"
+ "fmax v21.4s, v21.4s, v14.4s\n"
+ "fmax v22.4s, v22.4s, v14.4s\n"
+ "str q17, [%[c_ptr0], #0x10]\n"
+ "fmax v23.4s, v23.4s, v14.4s\n"
+ "fmin v20.4s, v20.4s, v15.4s\n"
+ "fmin v21.4s, v21.4s, v15.4s\n"
+ "str q18, [%[c_ptr0], #0x20]\n"
+ "fmin v22.4s, v22.4s, v15.4s\n"
+ "fmin v23.4s, v23.4s, v15.4s\n"
+ "fmax v24.4s, v24.4s, v14.4s\n"
+ "str q19, [%[c_ptr0], #0x30]\n"
+ "fmax v25.4s, v25.4s, v14.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x40\n"
+ "fmax v26.4s, v26.4s, v14.4s\n"
+ "str q20, [c_ptr1]\n"
+ "fmin v24.4s, v24.4s, v15.4s\n"
+ "fmin v25.4s, v25.4s, v15.4s\n"
+ "fmax v27.4s, v27.4s, v14.4s\n"
+ "str q21, [c_ptr1, #0x10]\n"
+ "fmin v26.4s, v26.4s, v15.4s\n"
+ "fmax v28.4s, v28.4s, v14.4s\n"
+ "fmax v29.4s, v29.4s, v14.4s\n"
+ "str q22, [c_ptr1, #0x20]\n"
+ "fmin v27.4s, v27.4s, v15.4s\n"
+ "fmax v30.4s, v30.4s, v14.4s\n"
+ "fmin v28.4s, v28.4s, v15.4s\n"
+ "str q23, [c_ptr1, #0x30]\n"
+ "fmin v29.4s, v29.4s, v15.4s\n"
+ "fmax v31.4s, v31.4s, v14.4s\n"
+ "fmin v30.4s, v30.4s, v15.4s\n"
+ "str q24, [c_ptr2]\n"
+ "fmin v31.4s, v31.4s, v15.4s\n"
+ "str q25, [c_ptr2, #0x10]\n"
+ "str q26, [c_ptr2, #0x20]\n"
+ "str q27, [c_ptr2, #0x30]\n"
+ "str q28, [c_ptr3]\n"
+ "str q29, [c_ptr3, #0x10]\n"
+ "str q30, [c_ptr3, #0x20]\n"
+ "str q31, [c_ptr3, #0x30]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
+ );
+ break;
+ }
+
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp
deleted file mode 100644
index 9b70827..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-namespace arm_gemm {
-
-// Actual kernel implementations
-void a64_sgemm_nativeA_pretransposeB_16x4(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int);
-
-// Native A/Pretranspose B SGEMM "strategy" class.
-//
-// This describes the characteristics of a family of kernels, in terms of
-// the required interleave properties and the output block size.
-//
-// All kernels in the family must share these characteristics. The actual
-// kernel to be used can be chosen at runtime, based on the CPUInfo
-// structure.
-class sgemm_nativeA_pretransposeB_16x4 {
-public:
- typedef float operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int);
-
- /* Desired data layout for B buffer (used for pretranspose) */
- static const int B_interleave = 16;
- static const int B_block = 1;
- static const bool B_transpose = true;
-
- /* Kernel blocking parameters */
- static unsigned int out_width() {
- return 16;
- }
-
- static unsigned int out_height() {
- return 4;
- }
-
- static unsigned int k_unroll() {
- return 1;
- }
-
- StdTransformsFixed<operand_type, result_type, 4, 16> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=a64_sgemm_nativeA_pretransposeB_16x4;
-
- sgemm_nativeA_pretransposeB_16x4(const CPUInfo *ci) {
- UNUSED(ci);
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp
deleted file mode 100644
index c51c8ff..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp
+++ /dev/null
@@ -1,970 +0,0 @@
-/*
- * Copyright (c) 2019 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-#include <algorithm>
-#include <cstddef>
-#include <cstring>
-
-#include <arm_neon.h>
-
-namespace arm_gemm {
-
-void a64_sgemm_nativeA_pretransposeB_16x4(const float *A, int lda, const float *B_panel, float *C, int ldc, float beta, unsigned int numrows, unsigned int numcols, unsigned int K) {
- const bool oddk = ((K % 8) >= 4);
- const bool beta0 = (beta == 0.0f);
- const unsigned int oddones = (K % 4);
-
- /* Use some small temporary arrays to cope with "ragged" M/N sizes.
- *
- * "dummy_A_buf" is used to avoid overreading the A input for ragged M,
- * and also for output if N is not ragged.
- *
- * Since the B input is pretransposed it will be padded as needed, so no
- * need to worry about overreading that.
- *
- * "C_buf" is used to avoid overreading or overwriting the output for
- * ragged N cases.
- */
- float dummy_A_buf[16];
- float C_buf[64];
-
- std::memset(dummy_A_buf, 0, sizeof(dummy_A_buf));
- std::memset(C_buf, 0, sizeof(C_buf));
-
- for (unsigned int y=0; y<numrows; y+=4) {
- const float *b_ptr = B_panel;
- const unsigned int active_rows = std::min(numrows - y, 4U);
-
- /* Increment values to be used to advance A pointers - these get set
- * to zero when the corresponding row isn't being used due to ragged
- * M, so it will just read the dummy buffer repeatedly. Values are
- * in bytes (8x sizeof(float)). */
- const unsigned long a_incr1 = (active_rows > 1) ? 32 : 0;
- const unsigned long a_incr2 = (active_rows > 2) ? 32 : 0;
- const unsigned long a_incr3 = (active_rows > 3) ? 32 : 0;
-
- /* Starting points for A pointers on this loop */
- const float * const a_ptr0_base = A + (y * lda);
- const float * const a_ptr1_base = (active_rows > 1) ? (a_ptr0_base + lda) : dummy_A_buf;
- const float * const a_ptr2_base = (active_rows > 2) ? (a_ptr1_base + lda) : dummy_A_buf;
- const float * const a_ptr3_base = (active_rows > 3) ? (a_ptr2_base + lda) : dummy_A_buf;
-
- /* Starting points for C pointers on this loop */
- float *c_ptr0 = C + (y * ldc);
- float *c_ptr1 = (active_rows > 1) ? (c_ptr0 + ldc) : dummy_A_buf;
- float *c_ptr2 = (active_rows > 2) ? (c_ptr1 + ldc) : dummy_A_buf;
- float *c_ptr3 = (active_rows > 3) ? (c_ptr2 + ldc) : dummy_A_buf;
-
- for (unsigned int x0=0; x0<numcols; x0+=16) {
- const unsigned int active_cols = std::min(numcols - x0, 16U);
- const bool use_result_buf = (active_cols < 16);
-
- /* Reset the A pointers for this loop. */
- const float *a_ptr0 = a_ptr0_base;
- const float *a_ptr1 = a_ptr1_base;
- const float *a_ptr2 = a_ptr2_base;
- const float *a_ptr3 = a_ptr3_base;
-
- /* Override C pointers if the result buffer is in use. */
- if (use_result_buf) {
- c_ptr0 = C_buf;
- c_ptr1 = C_buf + 16;
- c_ptr2 = C_buf + 32;
- c_ptr3 = C_buf + 48;
-
- /* If beta is non-zero, prepopulate the result buffer */
- if (!beta0) {
- for (unsigned int row=0; row<active_rows; row++) {
- for (unsigned int col=0; col<active_cols; col++) {
- C_buf[row * 16 + col] = C[((y + row) * ldc) + (x0 + col)];
- }
- }
- }
- }
-
- unsigned int loops = ((K+4)/8) - 1;
- unsigned int odds = oddones;
-
- __asm __volatile (
- "a0 .req v0\n"
- "a1 .req v1\n"
- "a2 .req v2\n"
- "a3 .req v3\n"
- "a0a .req v4\n"
- "a1a .req v5\n"
- "a2a .req v6\n"
- "a3a .req v7\n"
- "bb0 .req v8\n"
- "bb1 .req v9\n"
- "bb2 .req v10\n"
- "bb3 .req v11\n"
- "b0a .req v12\n"
- "b1a .req v13\n"
- "b2a .req v14\n"
- "b3a .req v15\n"
-
- "a0q .req q0\n"
- "a1q .req q1\n"
- "a2q .req q2\n"
- "a3q .req q3\n"
- "a0aq .req q4\n"
- "a1aq .req q5\n"
- "a2aq .req q6\n"
- "a3aq .req q7\n"
- "b0q .req q8\n"
- "b1q .req q9\n"
- "b2q .req q10\n"
- "b3q .req q11\n"
- "b0aq .req q12\n"
- "b1aq .req q13\n"
- "b2aq .req q14\n"
- "b3aq .req q15\n"
-
- "movi v16.4s, #0x0\n"
- "ldr a0q, [%[a_ptr0]]\n"
- "movi v17.4s, #0x0\n"
- "ldr b0q, [%[b_ptr]]\n"
- "movi v18.4s, #0x0\n"
- "ldr b1q, [%[b_ptr], #16]\n"
- "movi v19.4s, #0x0\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "movi v20.4s, #0x0\n"
- "ldr b3q, [%[b_ptr], #48]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "movi v21.4s, #0x0\n"
- "ldr a1q, [%[a_ptr1]]\n"
- "movi v22.4s, #0x0\n"
- "ldr a2q, [%[a_ptr2]]\n"
- "movi v23.4s, #0x0\n"
- "ldr a3q, [%[a_ptr3]]\n"
- "movi v24.4s, #0x0\n"
- "ldr b0aq, [%[b_ptr]]\n"
- "movi v25.4s, #0x0\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
- "movi v26.4s, #0x0\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
- "cbz %w[beta0], 5f\n"
- "movi v27.4s, #0x0\n"
- ASM_PREFETCH("[%[b_ptr], #0x40]")
- "movi v28.4s, #0x0\n"
- ASM_PREFETCH("[%[b_ptr], #0x80]")
- "movi v29.4s, #0x0\n"
- ASM_PREFETCH("[%[b_ptr], #0xC0]")
- "movi v30.4s, #0x0\n"
- ASM_PREFETCH("[%[b_ptr], #0x100]")
- "movi v31.4s, #0x0\n"
- ASM_PREFETCH("[%[b_ptr], #0x140]")
- ASM_PREFETCH("[%[b_ptr], #0x180]")
- ASM_PREFETCH("[%[b_ptr], #0x1C0]")
- ASM_PREFETCH("[%[b_ptr], #0x200]")
-
- // Skip if no complete loops.
- "cbz %w[loops], 4f\n"
- "b 1f\n"
-
- // If beta is non-zero, need to load and multiply by beta
- "5:\n"
- "ld1r {v4.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #16]\n"
- "ldr q18, [%[c_ptr0], #32]\n"
- "ldr q19, [%[c_ptr0], #48]\n"
-
- "ldr q20, [%[c_ptr1]]\n"
- "fmul v16.4s, v16.4s, v4.4s\n"
- "ldr q21, [%[c_ptr1], #16]\n"
- "fmul v17.4s, v17.4s, v4.4s\n"
- "ldr q22, [%[c_ptr1], #32]\n"
- "fmul v18.4s, v18.4s, v4.4s\n"
- "ldr q23, [%[c_ptr1], #48]\n"
- "fmul v19.4s, v19.4s, v4.4s\n"
-
- "ldr q24, [%[c_ptr2]]\n"
- "fmul v20.4s, v20.4s, v4.4s\n"
- "ldr q25, [%[c_ptr2], #16]\n"
- "fmul v21.4s, v21.4s, v4.4s\n"
- "ldr q26, [%[c_ptr2], #32]\n"
- "fmul v22.4s, v22.4s, v4.4s\n"
- "ldr q27, [%[c_ptr2], #48]\n"
- "fmul v23.4s, v23.4s, v4.4s\n"
-
- "ldr q28, [%[c_ptr3]]\n"
- "fmul v24.4s, v24.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0x40]")
- "ldr q29, [%[c_ptr3], #16]\n"
- "fmul v25.4s, v25.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0x80]")
- "ldr q30, [%[c_ptr3], #32]\n"
- "fmul v26.4s, v26.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0xC0]")
- "ldr q31, [%[c_ptr3], #48]\n"
- "fmul v27.4s, v27.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0x100]")
-
- "fmul v28.4s, v28.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0x140]")
- "fmul v29.4s, v29.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0x180]")
- "fmul v30.4s, v30.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0x1C0]")
- "fmul v31.4s, v31.4s, v4.4s\n"
- ASM_PREFETCH("[%[b_ptr], #0x200]")
-
- "cbz %w[loops], 4f\n"
-
- "1:\n"
- // Unroll 0
- "fmla v16.4s, bb0.4s, a0.s[0]\n"
- ASM_PREFETCH("[%[b_ptr], #0x240]")
- "fmla v20.4s, bb0.4s, a1.s[0]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
- "fmla v24.4s, bb0.4s, a2.s[0]\n"
- "fmla v28.4s, bb0.4s, a3.s[0]\n"
- "ldr b0q, [%[b_ptr], #64]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[0]\n"
- "fmla v21.4s, bb1.4s, a1.s[0]\n"
- "ldr a0aq, [%[a_ptr0], #16]\n"
- "fmla v25.4s, bb1.4s, a2.s[0]\n"
- "fmla v29.4s, bb1.4s, a3.s[0]\n"
- "ldr b1q, [%[b_ptr], #80]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[0]\n"
- "fmla v22.4s, bb2.4s, a1.s[0]\n"
- "ldr a1aq, [%[a_ptr1], #16]\n"
- "fmla v26.4s, bb2.4s, a2.s[0]\n"
- "fmla v30.4s, bb2.4s, a3.s[0]\n"
- "ldr b2q, [%[b_ptr], #96]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[0]\n"
- "fmla v23.4s, bb3.4s, a1.s[0]\n"
- "ldr a2aq, [%[a_ptr2], #16]\n"
- "fmla v27.4s, bb3.4s, a2.s[0]\n"
- "fmla v31.4s, bb3.4s, a3.s[0]\n"
- "ldr b3q, [%[b_ptr], #112]\n"
-
- // Unroll 1
- "fmla v16.4s, b0a.4s, a0.s[1]\n"
- ASM_PREFETCH("[%[b_ptr], #0x280]")
- "fmla v20.4s, b0a.4s, a1.s[1]\n"
- "ldr a3aq, [%[a_ptr3], #16]\n"
- "fmla v24.4s, b0a.4s, a2.s[1]\n"
- "fmla v28.4s, b0a.4s, a3.s[1]\n"
- "ldr b0aq, [%[b_ptr], #128]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[1]\n"
- "fmla v21.4s, b1a.4s, a1.s[1]\n"
- "subs %w[loops], %w[loops], #1\n"
- "fmla v25.4s, b1a.4s, a2.s[1]\n"
- "fmla v29.4s, b1a.4s, a3.s[1]\n"
- "ldr b1aq, [%[b_ptr], #144]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[1]\n"
- "fmla v22.4s, b2a.4s, a1.s[1]\n"
- "fmla v26.4s, b2a.4s, a2.s[1]\n"
- "fmla v30.4s, b2a.4s, a3.s[1]\n"
- "ldr b2aq, [%[b_ptr], #160]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[1]\n"
- "fmla v23.4s, b3a.4s, a1.s[1]\n"
- "fmla v27.4s, b3a.4s, a2.s[1]\n"
- "fmla v31.4s, b3a.4s, a3.s[1]\n"
- "ldr b3aq, [%[b_ptr], #176]\n"
-
- // Unroll 2
- "fmla v16.4s, bb0.4s, a0.s[2]\n"
- ASM_PREFETCH("[%[b_ptr], #0x2C0]")
- "fmla v20.4s, bb0.4s, a1.s[2]\n"
- "fmla v24.4s, bb0.4s, a2.s[2]\n"
- "fmla v28.4s, bb0.4s, a3.s[2]\n"
- "ldr b0q, [%[b_ptr], #192]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[2]\n"
- "add %[a_ptr0], %[a_ptr0], #32\n"
- "fmla v21.4s, bb1.4s, a1.s[2]\n"
- "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n"
- "fmla v25.4s, bb1.4s, a2.s[2]\n"
- "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n"
- "fmla v29.4s, bb1.4s, a3.s[2]\n"
- "ldr b1q, [%[b_ptr], #208]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[2]\n"
- "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n"
- "fmla v22.4s, bb2.4s, a1.s[2]\n"
- ASM_PREFETCH("[%[a_ptr0], #0x40]")
- "fmla v26.4s, bb2.4s, a2.s[2]\n"
- "fmla v30.4s, bb2.4s, a3.s[2]\n"
- "ldr b2q, [%[b_ptr], #224]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[2]\n"
- "fmla v23.4s, bb3.4s, a1.s[2]\n"
- ASM_PREFETCH("[%[a_ptr1], #0x40]")
- "fmla v27.4s, bb3.4s, a2.s[2]\n"
- "fmla v31.4s, bb3.4s, a3.s[2]\n"
- "ldr b3q, [%[b_ptr], #240]\n"
-
- // Unroll 3
- "fmla v16.4s, b0a.4s, a0.s[3]\n"
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "add %[b_ptr], %[b_ptr], #512\n"
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "ldr b0aq, [%[b_ptr], #-256]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[3]\n"
- ASM_PREFETCH("[%[b_ptr], #0x100]")
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "ldr b1aq, [%[b_ptr], #-240]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[3]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- ASM_PREFETCH("[%[a_ptr2], #0x40]")
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "ldr b2aq, [%[b_ptr], #-224]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[3]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "ldr a0q, [%[a_ptr0]]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
- "ldr b3aq, [%[b_ptr], #-208]\n"
-
- // Unroll 4
- "fmla v16.4s, bb0.4s, a0a.s[0]\n"
- "fmla v20.4s, bb0.4s, a1a.s[0]\n"
- ASM_PREFETCH("[%[b_ptr], #0x140]")
- "fmla v24.4s, bb0.4s, a2a.s[0]\n"
- "fmla v28.4s, bb0.4s, a3a.s[0]\n"
- "ldr b0q, [%[b_ptr], #-192]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[0]\n"
- "fmla v21.4s, bb1.4s, a1a.s[0]\n"
- "ldr a1q, [%[a_ptr1]]\n"
- "fmla v25.4s, bb1.4s, a2a.s[0]\n"
- "fmla v29.4s, bb1.4s, a3a.s[0]\n"
- "ldr b1q, [%[b_ptr], #-176]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[0]\n"
- "fmla v22.4s, bb2.4s, a1a.s[0]\n"
- "ldr a2q, [%[a_ptr2]]\n"
- "fmla v26.4s, bb2.4s, a2a.s[0]\n"
- "fmla v30.4s, bb2.4s, a3a.s[0]\n"
- "ldr b2q, [%[b_ptr], #-160]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[0]\n"
- "fmla v23.4s, bb3.4s, a1a.s[0]\n"
- "ldr a3q, [%[a_ptr3]]\n"
- "fmla v27.4s, bb3.4s, a2a.s[0]\n"
- "fmla v31.4s, bb3.4s, a3a.s[0]\n"
- "ldr b3q, [%[b_ptr], #-144]\n"
-
- // Unroll 5
- "fmla v16.4s, b0a.4s, a0a.s[1]\n"
- "fmla v20.4s, b0a.4s, a1a.s[1]\n"
- ASM_PREFETCH("[%[b_ptr], #0x180]")
- "fmla v24.4s, b0a.4s, a2a.s[1]\n"
- "fmla v28.4s, b0a.4s, a3a.s[1]\n"
- "ldr b0aq, [%[b_ptr], #-128]\n"
-
- "fmla v17.4s, b1a.4s, a0a.s[1]\n"
- "fmla v21.4s, b1a.4s, a1a.s[1]\n"
- ASM_PREFETCH("[%[a_ptr3], #0x40]")
- "fmla v25.4s, b1a.4s, a2a.s[1]\n"
- "fmla v29.4s, b1a.4s, a3a.s[1]\n"
- "ldr b1aq, [%[b_ptr], #-112]\n"
-
- "fmla v18.4s, b2a.4s, a0a.s[1]\n"
- "fmla v22.4s, b2a.4s, a1a.s[1]\n"
- "fmla v26.4s, b2a.4s, a2a.s[1]\n"
- "fmla v30.4s, b2a.4s, a3a.s[1]\n"
- "ldr b2aq, [%[b_ptr], #-96]\n"
-
- "fmla v19.4s, b3a.4s, a0a.s[1]\n"
- "fmla v23.4s, b3a.4s, a1a.s[1]\n"
- "fmla v27.4s, b3a.4s, a2a.s[1]\n"
- "fmla v31.4s, b3a.4s, a3a.s[1]\n"
- "ldr b3aq, [%[b_ptr], #-80]\n"
-
- // Unroll 6
- "fmla v16.4s, bb0.4s, a0a.s[2]\n"
- "fmla v20.4s, bb0.4s, a1a.s[2]\n"
- ASM_PREFETCH("[%[b_ptr], #0x1C0]")
- "fmla v24.4s, bb0.4s, a2a.s[2]\n"
- "fmla v28.4s, bb0.4s, a3a.s[2]\n"
- "ldr b0q, [%[b_ptr], #-64]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[2]\n"
- "fmla v21.4s, bb1.4s, a1a.s[2]\n"
- "fmla v25.4s, bb1.4s, a2a.s[2]\n"
- "fmla v29.4s, bb1.4s, a3a.s[2]\n"
- "ldr b1q, [%[b_ptr], #-48]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[2]\n"
- "fmla v22.4s, bb2.4s, a1a.s[2]\n"
- "fmla v26.4s, bb2.4s, a2a.s[2]\n"
- "fmla v30.4s, bb2.4s, a3a.s[2]\n"
- "ldr b2q, [%[b_ptr], #-32]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[2]\n"
- "fmla v23.4s, bb3.4s, a1a.s[2]\n"
- "fmla v27.4s, bb3.4s, a2a.s[2]\n"
- "fmla v31.4s, bb3.4s, a3a.s[2]\n"
- "ldr b3q, [%[b_ptr], #-16]\n"
-
- // Unroll 7
- "fmla v16.4s, b0a.4s, a0a.s[3]\n"
- "fmla v20.4s, b0a.4s, a1a.s[3]\n"
- "fmla v24.4s, b0a.4s, a2a.s[3]\n"
- "fmla v28.4s, b0a.4s, a3a.s[3]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0a.s[3]\n"
- "fmla v21.4s, b1a.4s, a1a.s[3]\n"
- ASM_PREFETCH("[%[b_ptr], #0x200]")
- "fmla v25.4s, b1a.4s, a2a.s[3]\n"
- "fmla v29.4s, b1a.4s, a3a.s[3]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0a.s[3]\n"
- "fmla v22.4s, b2a.4s, a1a.s[3]\n"
- "fmla v26.4s, b2a.4s, a2a.s[3]\n"
- "fmla v30.4s, b2a.4s, a3a.s[3]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0a.s[3]\n"
- "fmla v23.4s, b3a.4s, a1a.s[3]\n"
- "fmla v27.4s, b3a.4s, a2a.s[3]\n"
- "fmla v31.4s, b3a.4s, a3a.s[3]\n"
- "bne 1b\n"
-
- // Skip to here
- "4:\n"
-
- // Detached final iteration
- // Unroll 0
- "fmla v16.4s, bb0.4s, a0.s[0]\n"
- "fmla v20.4s, bb0.4s, a1.s[0]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
- "fmla v24.4s, bb0.4s, a2.s[0]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v28.4s, bb0.4s, a3.s[0]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[0]\n"
- "cbnz %w[oddk], 2f\n" // Deal with odd K before we load a0a
- "fmla v21.4s, bb1.4s, a1.s[0]\n"
- "ldr a0aq, [%[a_ptr0], #16]\n"
- "fmla v25.4s, bb1.4s, a2.s[0]\n"
- "fmla v29.4s, bb1.4s, a3.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[0]\n"
- "fmla v22.4s, bb2.4s, a1.s[0]\n"
- "ldr a1aq, [%[a_ptr1], #16]\n"
- "fmla v26.4s, bb2.4s, a2.s[0]\n"
- "fmla v30.4s, bb2.4s, a3.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[0]\n"
- "fmla v23.4s, bb3.4s, a1.s[0]\n"
- "ldr a2aq, [%[a_ptr2], #16]\n"
- "fmla v27.4s, bb3.4s, a2.s[0]\n"
- "fmla v31.4s, bb3.4s, a3.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 1
- "fmla v16.4s, b0a.4s, a0.s[1]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v20.4s, b0a.4s, a1.s[1]\n"
- "ldr a3aq, [%[a_ptr3], #16]\n"
- "fmla v24.4s, b0a.4s, a2.s[1]\n"
- "fmla v28.4s, b0a.4s, a3.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #32\n"
- "fmla v21.4s, b1a.4s, a1.s[1]\n"
- "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n"
- "fmla v25.4s, b1a.4s, a2.s[1]\n"
- "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n"
- "fmla v29.4s, b1a.4s, a3.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[1]\n"
- "fmla v22.4s, b2a.4s, a1.s[1]\n"
- "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n"
- "fmla v26.4s, b2a.4s, a2.s[1]\n"
- "fmla v30.4s, b2a.4s, a3.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[1]\n"
- "fmla v23.4s, b3a.4s, a1.s[1]\n"
- "fmla v27.4s, b3a.4s, a2.s[1]\n"
- "fmla v31.4s, b3a.4s, a3.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 2
- "fmla v16.4s, bb0.4s, a0.s[2]\n"
- "fmla v20.4s, bb0.4s, a1.s[2]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v24.4s, bb0.4s, a2.s[2]\n"
- "fmla v28.4s, bb0.4s, a3.s[2]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[2]\n"
- "fmla v21.4s, bb1.4s, a1.s[2]\n"
- "fmla v25.4s, bb1.4s, a2.s[2]\n"
- "fmla v29.4s, bb1.4s, a3.s[2]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[2]\n"
- "fmla v22.4s, bb2.4s, a1.s[2]\n"
- "fmla v26.4s, bb2.4s, a2.s[2]\n"
- "fmla v30.4s, bb2.4s, a3.s[2]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[2]\n"
- "fmla v23.4s, bb3.4s, a1.s[2]\n"
- "fmla v27.4s, bb3.4s, a2.s[2]\n"
- "fmla v31.4s, bb3.4s, a3.s[2]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 3
- "fmla v16.4s, b0a.4s, a0.s[3]\n"
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[3]\n"
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[3]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[3]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 4
- "fmla v16.4s, bb0.4s, a0a.s[0]\n"
- "fmla v20.4s, bb0.4s, a1a.s[0]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v24.4s, bb0.4s, a2a.s[0]\n"
- "fmla v28.4s, bb0.4s, a3a.s[0]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[0]\n"
- "fmla v21.4s, bb1.4s, a1a.s[0]\n"
- "fmla v25.4s, bb1.4s, a2a.s[0]\n"
- "fmla v29.4s, bb1.4s, a3a.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[0]\n"
- "fmla v22.4s, bb2.4s, a1a.s[0]\n"
- "fmla v26.4s, bb2.4s, a2a.s[0]\n"
- "fmla v30.4s, bb2.4s, a3a.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[0]\n"
- "fmla v23.4s, bb3.4s, a1a.s[0]\n"
- "fmla v27.4s, bb3.4s, a2a.s[0]\n"
- "fmla v31.4s, bb3.4s, a3a.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 5
- "fmla v16.4s, b0a.4s, a0a.s[1]\n"
- "fmla v20.4s, b0a.4s, a1a.s[1]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v24.4s, b0a.4s, a2a.s[1]\n"
- "fmla v28.4s, b0a.4s, a3a.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0a.s[1]\n"
- "fmla v21.4s, b1a.4s, a1a.s[1]\n"
- "fmla v25.4s, b1a.4s, a2a.s[1]\n"
- "fmla v29.4s, b1a.4s, a3a.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0a.s[1]\n"
- "fmla v22.4s, b2a.4s, a1a.s[1]\n"
- "fmla v26.4s, b2a.4s, a2a.s[1]\n"
- "fmla v30.4s, b2a.4s, a3a.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0a.s[1]\n"
- "fmla v23.4s, b3a.4s, a1a.s[1]\n"
- "fmla v27.4s, b3a.4s, a2a.s[1]\n"
- "fmla v31.4s, b3a.4s, a3a.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 6
- "fmla v16.4s, bb0.4s, a0a.s[2]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v20.4s, bb0.4s, a1a.s[2]\n"
- ASM_PREFETCH("[%[c_ptr0], #0x40]")
- "fmla v24.4s, bb0.4s, a2a.s[2]\n"
- "fmla v28.4s, bb0.4s, a3a.s[2]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[2]\n"
- "fmla v21.4s, bb1.4s, a1a.s[2]\n"
- ASM_PREFETCH("[%[c_ptr1], #0x40]")
- "fmla v25.4s, bb1.4s, a2a.s[2]\n"
- "fmla v29.4s, bb1.4s, a3a.s[2]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[2]\n"
- "fmla v22.4s, bb2.4s, a1a.s[2]\n"
- ASM_PREFETCH("[%[c_ptr2], #0x40]")
- "fmla v26.4s, bb2.4s, a2a.s[2]\n"
- "fmla v30.4s, bb2.4s, a3a.s[2]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[2]\n"
- "fmla v23.4s, bb3.4s, a1a.s[2]\n"
- ASM_PREFETCH("[%[c_ptr3], #0x40]")
- "fmla v27.4s, bb3.4s, a2a.s[2]\n"
- "fmla v31.4s, bb3.4s, a3a.s[2]\n"
-
- // Unroll 7
- "fmla v16.4s, b0a.4s, a0a.s[3]\n"
- "fmla v17.4s, b1a.4s, a0a.s[3]\n"
- "fmla v18.4s, b2a.4s, a0a.s[3]\n"
- "fmla v19.4s, b3a.4s, a0a.s[3]\n"
- "cbnz %w[odds], 6f\n"
-
- "fmla v20.4s, b0a.4s, a1a.s[3]\n"
- "str q16, [%[c_ptr0]]\n"
- "fmla v21.4s, b1a.4s, a1a.s[3]\n"
- "str q17, [%[c_ptr0], #16]\n"
- "fmla v22.4s, b2a.4s, a1a.s[3]\n"
- "str q18, [%[c_ptr0], #32]\n"
- "fmla v23.4s, b3a.4s, a1a.s[3]\n"
- "str q19, [%[c_ptr0], #48]\n"
-
- "fmla v24.4s, b0a.4s, a2a.s[3]\n"
- "str q20, [%[c_ptr1]]\n"
- "fmla v25.4s, b1a.4s, a2a.s[3]\n"
- "str q21, [%[c_ptr1], #16]\n"
- "fmla v26.4s, b2a.4s, a2a.s[3]\n"
- "str q22, [%[c_ptr1], #32]\n"
- "fmla v27.4s, b3a.4s, a2a.s[3]\n"
- "str q23, [%[c_ptr1], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3a.s[3]\n"
- "str q24, [%[c_ptr2]]\n"
- "fmla v29.4s, b1a.4s, a3a.s[3]\n"
- "str q25, [%[c_ptr2], #16]\n"
- "fmla v30.4s, b2a.4s, a3a.s[3]\n"
- "str q26, [%[c_ptr2], #32]\n"
- "fmla v31.4s, b3a.4s, a3a.s[3]\n"
- "str q27, [%[c_ptr2], #48]\n"
- "b 3f\n"
-
- // Odd K case: Just do 4 more.
- "2:\n"
- "fmla v21.4s, bb1.4s, a1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #16\n"
- "fmla v25.4s, bb1.4s, a2.s[0]\n"
- "add %[a_ptr1], %[a_ptr1], #16\n"
- "fmla v29.4s, bb1.4s, a3.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[0]\n"
- "add %[a_ptr2], %[a_ptr2], #16\n"
- "fmla v22.4s, bb2.4s, a1.s[0]\n"
- "add %[a_ptr3], %[a_ptr3], #16\n"
- "fmla v26.4s, bb2.4s, a2.s[0]\n"
- "fmla v30.4s, bb2.4s, a3.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[0]\n"
- "fmla v23.4s, bb3.4s, a1.s[0]\n"
- "fmla v27.4s, bb3.4s, a2.s[0]\n"
- "fmla v31.4s, bb3.4s, a3.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 1
- "fmla v16.4s, b0a.4s, a0.s[1]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v20.4s, b0a.4s, a1.s[1]\n"
- "fmla v24.4s, b0a.4s, a2.s[1]\n"
- "fmla v28.4s, b0a.4s, a3.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[1]\n"
- "fmla v21.4s, b1a.4s, a1.s[1]\n"
- "fmla v25.4s, b1a.4s, a2.s[1]\n"
- "fmla v29.4s, b1a.4s, a3.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[1]\n"
- "fmla v22.4s, b2a.4s, a1.s[1]\n"
- "fmla v26.4s, b2a.4s, a2.s[1]\n"
- "fmla v30.4s, b2a.4s, a3.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[1]\n"
- "fmla v23.4s, b3a.4s, a1.s[1]\n"
- "fmla v27.4s, b3a.4s, a2.s[1]\n"
- "fmla v31.4s, b3a.4s, a3.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 2
- "fmla v16.4s, bb0.4s, a0.s[2]\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v20.4s, bb0.4s, a1.s[2]\n"
- ASM_PREFETCH("[%[c_ptr0], #0x40]")
- "fmla v24.4s, bb0.4s, a2.s[2]\n"
- "fmla v28.4s, bb0.4s, a3.s[2]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[2]\n"
- "fmla v21.4s, bb1.4s, a1.s[2]\n"
- ASM_PREFETCH("[%[c_ptr1], #0x40]")
- "fmla v25.4s, bb1.4s, a2.s[2]\n"
- "fmla v29.4s, bb1.4s, a3.s[2]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[2]\n"
- "fmla v22.4s, bb2.4s, a1.s[2]\n"
- ASM_PREFETCH("[%[c_ptr2], #0x40]")
- "fmla v26.4s, bb2.4s, a2.s[2]\n"
- "fmla v30.4s, bb2.4s, a3.s[2]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[2]\n"
- "fmla v23.4s, bb3.4s, a1.s[2]\n"
- ASM_PREFETCH("[%[c_ptr3], #0x40]")
- "fmla v27.4s, bb3.4s, a2.s[2]\n"
- "fmla v31.4s, bb3.4s, a3.s[2]\n"
-
- // Unroll 3
- "fmla v16.4s, b0a.4s, a0.s[3]\n"
- "fmla v17.4s, b1a.4s, a0.s[3]\n"
- "fmla v18.4s, b2a.4s, a0.s[3]\n"
- "fmla v19.4s, b3a.4s, a0.s[3]\n"
- "cbnz %w[odds], 7f\n"
-
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "str q16, [%[c_ptr0]]\n"
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "str q17, [%[c_ptr0], #16]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- "str q18, [%[c_ptr0], #32]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "str q19, [%[c_ptr0], #48]\n"
-
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "str q20, [%[c_ptr1]]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "str q21, [%[c_ptr1], #16]\n"
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "str q22, [%[c_ptr1], #32]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "str q23, [%[c_ptr1], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "str q24, [%[c_ptr2]]\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "str q25, [%[c_ptr2], #16]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "str q26, [%[c_ptr2], #32]\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
- "str q27, [%[c_ptr2], #48]\n"
- "b 3f\n"
-
- // "Odd ones" - lead in from even
- "6:\n"
- "fmla v20.4s, b0a.4s, a1a.s[3]\n"
- "fmla v21.4s, b1a.4s, a1a.s[3]\n"
- "ldr b0q, [%[b_ptr]]\n"
- "fmla v22.4s, b2a.4s, a1a.s[3]\n"
- "subs %w[odds], %w[odds], #1\n"
- "fmla v23.4s, b3a.4s, a1a.s[3]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v24.4s, b0a.4s, a2a.s[3]\n"
- "fmla v25.4s, b1a.4s, a2a.s[3]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "fmla v26.4s, b2a.4s, a2a.s[3]\n"
- "fmla v27.4s, b3a.4s, a2a.s[3]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3a.s[3]\n"
- "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
- "fmla v29.4s, b1a.4s, a3a.s[3]\n"
- "fmla v30.4s, b2a.4s, a3a.s[3]\n"
- "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
- "fmla v31.4s, b3a.4s, a3a.s[3]\n"
-
- "fmla v16.4s, bb0.4s, a0.4s\n"
- "beq 9f\n"
- "b 8f\n"
-
- // "Odd ones" - lead in from odd
- "7:\n"
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "subs %w[odds], %w[odds], #1\n"
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "ldr b0q, [%[b_ptr]]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
-
- "fmla v16.4s, bb0.4s, a0.4s\n"
- "beq 9f\n"
-
- // "Odd ones" - loop
- "8:\n"
- "fmla v17.4s, bb1.4s, a0.4s\n"
- "ld1r {a2.4s}, [%[a_ptr2]], #4\n"
- "fmla v18.4s, bb2.4s, a0.4s\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v19.4s, bb3.4s, a0.4s\n"
- "ld1r {a3.4s}, [%[a_ptr3]], #4\n"
-
- "fmla v20.4s, bb0.4s, a1.4s\n"
- "subs %w[odds], %w[odds], #1\n"
- "fmla v21.4s, bb1.4s, a1.4s\n"
- "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
- "fmla v22.4s, bb2.4s, a1.4s\n"
- "fmla v23.4s, bb3.4s, a1.4s\n"
- "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
-
- "fmla v24.4s, bb0.4s, a2.4s\n"
- "fmla v28.4s, bb0.4s, a3.4s\n"
- "ldr b0q, [%[b_ptr]]\n"
- "fmla v25.4s, bb1.4s, a2.4s\n"
- "fmla v29.4s, bb1.4s, a3.4s\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v26.4s, bb2.4s, a2.4s\n"
- "fmla v30.4s, bb2.4s, a3.4s\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "fmla v27.4s, bb3.4s, a2.4s\n"
- "fmla v31.4s, bb3.4s, a3.4s\n"
- "ldr b3q, [%[b_ptr], #48]\n"
- "fmla v16.4s, bb0.4s, a0.4s\n"
- "bne 8b\n"
-
- // "Odd ones" - detached final iteration
- "9:\n"
- "fmla v17.4s, bb1.4s, a0.4s\n"
- "ld1r {a2.4s}, [%[a_ptr2]], #4\n"
- "fmla v18.4s, bb2.4s, a0.4s\n"
- "add %[b_ptr], %[b_ptr], #64\n"
- "fmla v19.4s, bb3.4s, a0.4s\n"
- "ld1r {a3.4s}, [%[a_ptr3]], #4\n"
-
- "fmla v20.4s, bb0.4s, a1.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmla v21.4s, bb1.4s, a1.4s\n"
- "str q17, [%[c_ptr0], #16]\n"
- "fmla v22.4s, bb2.4s, a1.4s\n"
- "str q18, [%[c_ptr0], #32]\n"
- "fmla v23.4s, bb3.4s, a1.4s\n"
- "str q19, [%[c_ptr0], #48]\n"
-
- "fmla v24.4s, bb0.4s, a2.4s\n"
- "str q20, [%[c_ptr1]]\n"
- "fmla v25.4s, bb1.4s, a2.4s\n"
- "str q21, [%[c_ptr1], #16]\n"
- "fmla v26.4s, bb2.4s, a2.4s\n"
- "str q22, [%[c_ptr1], #32]\n"
- "fmla v27.4s, bb3.4s, a2.4s\n"
- "str q23, [%[c_ptr1], #48]\n"
-
- "fmla v28.4s, bb0.4s, a3.4s\n"
- "str q24, [%[c_ptr2]]\n"
- "fmla v29.4s, bb1.4s, a3.4s\n"
- "str q25, [%[c_ptr2], #16]\n"
- "fmla v30.4s, bb2.4s, a3.4s\n"
- "str q26, [%[c_ptr2], #32]\n"
- "fmla v31.4s, bb3.4s, a3.4s\n"
- "str q27, [%[c_ptr2], #48]\n"
-
- "3:\n"
- "str q28, [%[c_ptr3]]\n"
- // Increment C pointers for next loop - this looks odd if we
- // are using the result buffer, but it's OK as using the
- // result buffer implies there will be no next loop.
- "add %[c_ptr0], %[c_ptr0], #64\n"
- "str q29, [%[c_ptr3], #16]\n"
- "add %[c_ptr1], %[c_ptr1], %[a_incr1], LSL #1\n"
- "str q30, [%[c_ptr3], #32]\n"
- "add %[c_ptr2], %[c_ptr2], %[a_incr2], LSL #1\n"
- "str q31, [%[c_ptr3], #48]\n"
- "add %[c_ptr3], %[c_ptr3], %[a_incr3], LSL #1\n"
-
- : [a_ptr0] "+r" (a_ptr0), [a_ptr1] "+r" (a_ptr1), [a_ptr2] "+r" (a_ptr2), [a_ptr3] "+r" (a_ptr3),
- [b_ptr] "+r" (b_ptr), [loops] "+r" (loops), [odds] "+r" (odds),
- [c_ptr0] "+r" (c_ptr0), [c_ptr1] "+r" (c_ptr1), [c_ptr2] "+r" (c_ptr2), [c_ptr3] "+r" (c_ptr3)
- : [oddk] "r" (oddk), [beta0] "r" (beta0), [betaptr] "r" (&beta),
- [a_incr1] "r" (a_incr1), [a_incr2] "r" (a_incr2), [a_incr3] "r" (a_incr3)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
- "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
- "cc", "memory"
- );
-
- /* Copy results from result buffer if needed. */
- if (use_result_buf) {
- for (unsigned int row=0; row<active_rows; row++) {
- for (unsigned int col=0; col<active_cols; col++) {
- C[((y + row) * ldc) + (x0 + col)] = C_buf[row * 16 + col];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp
deleted file mode 100644
index a551193..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2017-2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-namespace arm_gemm {
-
-// Actual kernel implementations
-void a64_sgemm_native_16x4(const float *, int, const float *, int, float *, int, float, int, int, int);
-
-// 12x8 SGEMM "strategy" class.
-//
-// This describes the characteristics of a family of kernels, in terms of
-// the required interleave properties and the output block size.
-//
-// All kernels in the family must share these characteristics. The actual
-// kernel to be used can be chosen at runtime, based on the CPU_type
-// structure.
-class sgemm_native_16x4 {
-public:
- typedef float operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const float *, int, const float *, int, float *, int, float, int, int, int);
-
- /* Kernel blocking parameters */
- static unsigned int out_width() {
- return 16;
- }
-
- static unsigned int out_height() {
- return 4;
- }
-
- static unsigned int k_unroll() {
- return 1;
- }
-
- // Default to the generic kernel
- kern_type kernel=a64_sgemm_native_16x4;
-
- sgemm_native_16x4(const CPUInfo *ci) {
- UNUSED(ci);
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp
deleted file mode 100644
index 8325b3f..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp
+++ /dev/null
@@ -1,889 +0,0 @@
-/*
- * Copyright (c) 2017-2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-#include <cstddef>
-#include <cstring>
-
-#include <arm_neon.h>
-
-namespace arm_gemm {
-
-void a64_sgemm_native_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) {
- const int oddk = ((K % 8) >= 4) ? 1 : 0;
- const int beta0 = (beta == 0.0f) ? 1 : 0;
- const int oddones = (K % 4);
-
- float dummy_buffer[16];
-
- std::memset(dummy_buffer, 0, sizeof(dummy_buffer));
-
- /* For now, very naive with no blocking */
- for (int y=0; y<M; y+=4) {
- const int activerows = std::min(M-y, 4);
-
- const float * const a_ptr0_base = A + (y * lda);
- const float * const a_ptr1_base = (activerows > 1) ? (a_ptr0_base + lda) : dummy_buffer;
- const float * const a_ptr2_base = (activerows > 2) ? (a_ptr1_base + lda) : dummy_buffer;
- const float * const a_ptr3_base = (activerows > 3) ? (a_ptr2_base + lda) : dummy_buffer;
-
- const unsigned long a_incr1 = (activerows > 1) ? 32 : 0;
- const unsigned long a_incr2 = (activerows > 2) ? 32 : 0;
- const unsigned long a_incr3 = (activerows > 3) ? 32 : 0;
-
- float *c_ptr0 = C + (y * ldc);
- float *c_ptr1 = (activerows > 1) ? c_ptr0 + ldc : dummy_buffer;
- float *c_ptr2 = (activerows > 2) ? c_ptr1 + ldc : dummy_buffer;
- float *c_ptr3 = (activerows > 3) ? c_ptr2 + ldc : dummy_buffer;
-
- for (int x0=0; x0<N; x0+=16) {
- const float *a_ptr0 = a_ptr0_base;
- const float *a_ptr1 = a_ptr1_base;
- const float *a_ptr2 = a_ptr2_base;
- const float *a_ptr3 = a_ptr3_base;
-
- const float *b_ptr = B + x0;
-
- int loops = ((K+4)/8) - 1;
- int odds = oddones;
-
- size_t ldbb = ldb * sizeof(float);
-
- __asm __volatile (
- "a0 .req v0\n"
- "a1 .req v1\n"
- "a2 .req v2\n"
- "a3 .req v3\n"
- "a0a .req v4\n"
- "a1a .req v5\n"
- "a2a .req v6\n"
- "a3a .req v7\n"
- "bb0 .req v8\n"
- "bb1 .req v9\n"
- "bb2 .req v10\n"
- "bb3 .req v11\n"
- "b0a .req v12\n"
- "b1a .req v13\n"
- "b2a .req v14\n"
- "b3a .req v15\n"
-
- "a0q .req q0\n"
- "a1q .req q1\n"
- "a2q .req q2\n"
- "a3q .req q3\n"
- "a0aq .req q4\n"
- "a1aq .req q5\n"
- "a2aq .req q6\n"
- "a3aq .req q7\n"
- "b0q .req q8\n"
- "b1q .req q9\n"
- "b2q .req q10\n"
- "b3q .req q11\n"
- "b0aq .req q12\n"
- "b1aq .req q13\n"
- "b2aq .req q14\n"
- "b3aq .req q15\n"
-
- "movi v16.4s, #0x0\n"
- "ldr a0q, [%[a_ptr0]]\n"
- "movi v17.4s, #0x0\n"
- "ldr b0q, [%[b_ptr]]\n"
- "movi v18.4s, #0x0\n"
- "ldr b1q, [%[b_ptr], #16]\n"
- "movi v19.4s, #0x0\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "movi v20.4s, #0x0\n"
- "ldr b3q, [%[b_ptr], #48]\n"
- "movi v21.4s, #0x0\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "ldr a1q, [%[a_ptr1]]\n"
- "movi v22.4s, #0x0\n"
- "ldr a2q, [%[a_ptr2]]\n"
- "movi v23.4s, #0x0\n"
- "ldr a3q, [%[a_ptr3]]\n"
- "movi v24.4s, #0x0\n"
- "ldr b0aq, [%[b_ptr]]\n"
- "movi v25.4s, #0x0\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
- "movi v26.4s, #0x0\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
- "cbz %w[beta0], 5f\n"
- "movi v27.4s, #0x0\n"
- "movi v28.4s, #0x0\n"
- "movi v29.4s, #0x0\n"
- "movi v30.4s, #0x0\n"
- "movi v31.4s, #0x0\n"
-
- // Skip if no complete loops.
- "cbz %w[loops], 4f\n"
- "b 1f\n"
-
- // If beta is non-zero, need to load and multiply by beta
- "5:\n"
- "ld1r {v4.4s}, [%[betaptr]]\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #16]\n"
- "ldr q18, [%[c_ptr0], #32]\n"
- "ldr q19, [%[c_ptr0], #48]\n"
-
- "ldr q20, [%[c_ptr1]]\n"
- "fmul v16.4s, v16.4s, v4.4s\n"
- "ldr q21, [%[c_ptr1], #16]\n"
- "fmul v17.4s, v17.4s, v4.4s\n"
- "ldr q22, [%[c_ptr1], #32]\n"
- "fmul v18.4s, v18.4s, v4.4s\n"
- "ldr q23, [%[c_ptr1], #48]\n"
- "fmul v19.4s, v19.4s, v4.4s\n"
-
- "ldr q24, [%[c_ptr2]]\n"
- "fmul v20.4s, v20.4s, v4.4s\n"
- "ldr q25, [%[c_ptr2], #16]\n"
- "fmul v21.4s, v21.4s, v4.4s\n"
- "ldr q26, [%[c_ptr2], #32]\n"
- "fmul v22.4s, v22.4s, v4.4s\n"
- "ldr q27, [%[c_ptr2], #48]\n"
- "fmul v23.4s, v23.4s, v4.4s\n"
-
- "ldr q28, [%[c_ptr3]]\n"
- "fmul v24.4s, v24.4s, v4.4s\n"
- "ldr q29, [%[c_ptr3], #16]\n"
- "fmul v25.4s, v25.4s, v4.4s\n"
- "ldr q30, [%[c_ptr3], #32]\n"
- "fmul v26.4s, v26.4s, v4.4s\n"
- "ldr q31, [%[c_ptr3], #48]\n"
- "fmul v27.4s, v27.4s, v4.4s\n"
-
- "fmul v28.4s, v28.4s, v4.4s\n"
- "fmul v29.4s, v29.4s, v4.4s\n"
- "fmul v30.4s, v30.4s, v4.4s\n"
- "fmul v31.4s, v31.4s, v4.4s\n"
-
- "cbz %w[loops], 4f\n"
-
- "1:\n"
- // Unroll 0
- "fmla v16.4s, bb0.4s, a0.s[0]\n"
- "fmla v20.4s, bb0.4s, a1.s[0]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
- "fmla v24.4s, bb0.4s, a2.s[0]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v28.4s, bb0.4s, a3.s[0]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[0]\n"
- "fmla v21.4s, bb1.4s, a1.s[0]\n"
- "ldr a0aq, [%[a_ptr0], #16]\n"
- "fmla v25.4s, bb1.4s, a2.s[0]\n"
- "fmla v29.4s, bb1.4s, a3.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[0]\n"
- "fmla v22.4s, bb2.4s, a1.s[0]\n"
- "ldr a1aq, [%[a_ptr1], #16]\n"
- "fmla v26.4s, bb2.4s, a2.s[0]\n"
- "fmla v30.4s, bb2.4s, a3.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[0]\n"
- "fmla v23.4s, bb3.4s, a1.s[0]\n"
- "ldr a2aq, [%[a_ptr2], #16]\n"
- "fmla v27.4s, bb3.4s, a2.s[0]\n"
- "fmla v31.4s, bb3.4s, a3.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 1
- "fmla v16.4s, b0a.4s, a0.s[1]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v20.4s, b0a.4s, a1.s[1]\n"
- "ldr a3aq, [%[a_ptr3], #16]\n"
- "fmla v24.4s, b0a.4s, a2.s[1]\n"
- "fmla v28.4s, b0a.4s, a3.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[1]\n"
- "fmla v21.4s, b1a.4s, a1.s[1]\n"
- "subs %w[loops], %w[loops], #1\n"
- "fmla v25.4s, b1a.4s, a2.s[1]\n"
- "fmla v29.4s, b1a.4s, a3.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[1]\n"
- "fmla v22.4s, b2a.4s, a1.s[1]\n"
- "fmla v26.4s, b2a.4s, a2.s[1]\n"
- "fmla v30.4s, b2a.4s, a3.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[1]\n"
- "fmla v23.4s, b3a.4s, a1.s[1]\n"
- "fmla v27.4s, b3a.4s, a2.s[1]\n"
- "fmla v31.4s, b3a.4s, a3.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 2
- "fmla v16.4s, bb0.4s, a0.s[2]\n"
- "fmla v20.4s, bb0.4s, a1.s[2]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, bb0.4s, a2.s[2]\n"
- "fmla v28.4s, bb0.4s, a3.s[2]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[2]\n"
- "add %[a_ptr0], %[a_ptr0], #32\n"
- "fmla v21.4s, bb1.4s, a1.s[2]\n"
- "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n"
- "fmla v25.4s, bb1.4s, a2.s[2]\n"
- "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n"
- "fmla v29.4s, bb1.4s, a3.s[2]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[2]\n"
- "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n"
- "fmla v22.4s, bb2.4s, a1.s[2]\n"
- "fmla v26.4s, bb2.4s, a2.s[2]\n"
- "fmla v30.4s, bb2.4s, a3.s[2]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[2]\n"
- "fmla v23.4s, bb3.4s, a1.s[2]\n"
- "fmla v27.4s, bb3.4s, a2.s[2]\n"
- "fmla v31.4s, bb3.4s, a3.s[2]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 3
- "fmla v16.4s, b0a.4s, a0.s[3]\n"
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[3]\n"
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[3]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[3]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "ldr a0q, [%[a_ptr0]]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 4
- "fmla v16.4s, bb0.4s, a0a.s[0]\n"
- "fmla v20.4s, bb0.4s, a1a.s[0]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, bb0.4s, a2a.s[0]\n"
- "fmla v28.4s, bb0.4s, a3a.s[0]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[0]\n"
- "fmla v21.4s, bb1.4s, a1a.s[0]\n"
- "ldr a1q, [%[a_ptr1]]\n"
- "fmla v25.4s, bb1.4s, a2a.s[0]\n"
- "fmla v29.4s, bb1.4s, a3a.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[0]\n"
- "fmla v22.4s, bb2.4s, a1a.s[0]\n"
- "ldr a2q, [%[a_ptr2]]\n"
- "fmla v26.4s, bb2.4s, a2a.s[0]\n"
- "fmla v30.4s, bb2.4s, a3a.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[0]\n"
- "fmla v23.4s, bb3.4s, a1a.s[0]\n"
- "ldr a3q, [%[a_ptr3]]\n"
- "fmla v27.4s, bb3.4s, a2a.s[0]\n"
- "fmla v31.4s, bb3.4s, a3a.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 5
- "fmla v16.4s, b0a.4s, a0a.s[1]\n"
- "fmla v20.4s, b0a.4s, a1a.s[1]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, b0a.4s, a2a.s[1]\n"
- "fmla v28.4s, b0a.4s, a3a.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0a.s[1]\n"
- "fmla v21.4s, b1a.4s, a1a.s[1]\n"
- "fmla v25.4s, b1a.4s, a2a.s[1]\n"
- "fmla v29.4s, b1a.4s, a3a.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0a.s[1]\n"
- "fmla v22.4s, b2a.4s, a1a.s[1]\n"
- "fmla v26.4s, b2a.4s, a2a.s[1]\n"
- "fmla v30.4s, b2a.4s, a3a.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0a.s[1]\n"
- "fmla v23.4s, b3a.4s, a1a.s[1]\n"
- "fmla v27.4s, b3a.4s, a2a.s[1]\n"
- "fmla v31.4s, b3a.4s, a3a.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 6
- "fmla v16.4s, bb0.4s, a0a.s[2]\n"
- "fmla v20.4s, bb0.4s, a1a.s[2]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, bb0.4s, a2a.s[2]\n"
- "fmla v28.4s, bb0.4s, a3a.s[2]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[2]\n"
- "fmla v21.4s, bb1.4s, a1a.s[2]\n"
- "fmla v25.4s, bb1.4s, a2a.s[2]\n"
- "fmla v29.4s, bb1.4s, a3a.s[2]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[2]\n"
- "fmla v22.4s, bb2.4s, a1a.s[2]\n"
- "fmla v26.4s, bb2.4s, a2a.s[2]\n"
- "fmla v30.4s, bb2.4s, a3a.s[2]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[2]\n"
- "fmla v23.4s, bb3.4s, a1a.s[2]\n"
- "fmla v27.4s, bb3.4s, a2a.s[2]\n"
- "fmla v31.4s, bb3.4s, a3a.s[2]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 7
- "fmla v16.4s, b0a.4s, a0a.s[3]\n"
- "fmla v20.4s, b0a.4s, a1a.s[3]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, b0a.4s, a2a.s[3]\n"
- "fmla v28.4s, b0a.4s, a3a.s[3]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0a.s[3]\n"
- "fmla v21.4s, b1a.4s, a1a.s[3]\n"
- "fmla v25.4s, b1a.4s, a2a.s[3]\n"
- "fmla v29.4s, b1a.4s, a3a.s[3]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0a.s[3]\n"
- "fmla v22.4s, b2a.4s, a1a.s[3]\n"
- "fmla v26.4s, b2a.4s, a2a.s[3]\n"
- "fmla v30.4s, b2a.4s, a3a.s[3]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0a.s[3]\n"
- "fmla v23.4s, b3a.4s, a1a.s[3]\n"
- "fmla v27.4s, b3a.4s, a2a.s[3]\n"
- "fmla v31.4s, b3a.4s, a3a.s[3]\n"
- "bne 1b\n"
-
- // Skip to here
- "4:\n"
-
- // Detached final iteration
- // Unroll 0
- "fmla v16.4s, bb0.4s, a0.s[0]\n"
- "fmla v20.4s, bb0.4s, a1.s[0]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
- "fmla v24.4s, bb0.4s, a2.s[0]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v28.4s, bb0.4s, a3.s[0]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[0]\n"
- "cbnz %w[oddk], 2f\n" // Deal with odd K before we load a0a
- "fmla v21.4s, bb1.4s, a1.s[0]\n"
- "ldr a0aq, [%[a_ptr0], #16]\n"
- "fmla v25.4s, bb1.4s, a2.s[0]\n"
- "fmla v29.4s, bb1.4s, a3.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[0]\n"
- "fmla v22.4s, bb2.4s, a1.s[0]\n"
- "ldr a1aq, [%[a_ptr1], #16]\n"
- "fmla v26.4s, bb2.4s, a2.s[0]\n"
- "fmla v30.4s, bb2.4s, a3.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[0]\n"
- "fmla v23.4s, bb3.4s, a1.s[0]\n"
- "ldr a2aq, [%[a_ptr2], #16]\n"
- "fmla v27.4s, bb3.4s, a2.s[0]\n"
- "fmla v31.4s, bb3.4s, a3.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 1
- "fmla v16.4s, b0a.4s, a0.s[1]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v20.4s, b0a.4s, a1.s[1]\n"
- "ldr a3aq, [%[a_ptr3], #16]\n"
- "fmla v24.4s, b0a.4s, a2.s[1]\n"
- "fmla v28.4s, b0a.4s, a3.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #32\n"
- "fmla v21.4s, b1a.4s, a1.s[1]\n"
- "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n"
- "fmla v25.4s, b1a.4s, a2.s[1]\n"
- "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n"
- "fmla v29.4s, b1a.4s, a3.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[1]\n"
- "fmla v22.4s, b2a.4s, a1.s[1]\n"
- "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n"
- "fmla v26.4s, b2a.4s, a2.s[1]\n"
- "fmla v30.4s, b2a.4s, a3.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[1]\n"
- "fmla v23.4s, b3a.4s, a1.s[1]\n"
- "fmla v27.4s, b3a.4s, a2.s[1]\n"
- "fmla v31.4s, b3a.4s, a3.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 2
- "fmla v16.4s, bb0.4s, a0.s[2]\n"
- "fmla v20.4s, bb0.4s, a1.s[2]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, bb0.4s, a2.s[2]\n"
- "fmla v28.4s, bb0.4s, a3.s[2]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[2]\n"
- "fmla v21.4s, bb1.4s, a1.s[2]\n"
- "fmla v25.4s, bb1.4s, a2.s[2]\n"
- "fmla v29.4s, bb1.4s, a3.s[2]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[2]\n"
- "fmla v22.4s, bb2.4s, a1.s[2]\n"
- "fmla v26.4s, bb2.4s, a2.s[2]\n"
- "fmla v30.4s, bb2.4s, a3.s[2]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[2]\n"
- "fmla v23.4s, bb3.4s, a1.s[2]\n"
- "fmla v27.4s, bb3.4s, a2.s[2]\n"
- "fmla v31.4s, bb3.4s, a3.s[2]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 3
- "fmla v16.4s, b0a.4s, a0.s[3]\n"
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[3]\n"
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[3]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[3]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 4
- "fmla v16.4s, bb0.4s, a0a.s[0]\n"
- "fmla v20.4s, bb0.4s, a1a.s[0]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, bb0.4s, a2a.s[0]\n"
- "fmla v28.4s, bb0.4s, a3a.s[0]\n"
- "ldr b0q, [%[b_ptr]]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[0]\n"
- "fmla v21.4s, bb1.4s, a1a.s[0]\n"
- "fmla v25.4s, bb1.4s, a2a.s[0]\n"
- "fmla v29.4s, bb1.4s, a3a.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[0]\n"
- "fmla v22.4s, bb2.4s, a1a.s[0]\n"
- "fmla v26.4s, bb2.4s, a2a.s[0]\n"
- "fmla v30.4s, bb2.4s, a3a.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[0]\n"
- "fmla v23.4s, bb3.4s, a1a.s[0]\n"
- "fmla v27.4s, bb3.4s, a2a.s[0]\n"
- "fmla v31.4s, bb3.4s, a3a.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 5
- "fmla v16.4s, b0a.4s, a0a.s[1]\n"
- "fmla v20.4s, b0a.4s, a1a.s[1]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, b0a.4s, a2a.s[1]\n"
- "fmla v28.4s, b0a.4s, a3a.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0a.s[1]\n"
- "fmla v21.4s, b1a.4s, a1a.s[1]\n"
- "fmla v25.4s, b1a.4s, a2a.s[1]\n"
- "fmla v29.4s, b1a.4s, a3a.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0a.s[1]\n"
- "fmla v22.4s, b2a.4s, a1a.s[1]\n"
- "fmla v26.4s, b2a.4s, a2a.s[1]\n"
- "fmla v30.4s, b2a.4s, a3a.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0a.s[1]\n"
- "fmla v23.4s, b3a.4s, a1a.s[1]\n"
- "fmla v27.4s, b3a.4s, a2a.s[1]\n"
- "fmla v31.4s, b3a.4s, a3a.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 6
- "fmla v16.4s, bb0.4s, a0a.s[2]\n"
- "fmla v20.4s, bb0.4s, a1a.s[2]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v24.4s, bb0.4s, a2a.s[2]\n"
- "fmla v28.4s, bb0.4s, a3a.s[2]\n"
-
- "fmla v17.4s, bb1.4s, a0a.s[2]\n"
- "fmla v21.4s, bb1.4s, a1a.s[2]\n"
- "fmla v25.4s, bb1.4s, a2a.s[2]\n"
- "fmla v29.4s, bb1.4s, a3a.s[2]\n"
-
- "fmla v18.4s, bb2.4s, a0a.s[2]\n"
- "fmla v22.4s, bb2.4s, a1a.s[2]\n"
- "fmla v26.4s, bb2.4s, a2a.s[2]\n"
- "fmla v30.4s, bb2.4s, a3a.s[2]\n"
-
- "fmla v19.4s, bb3.4s, a0a.s[2]\n"
- "fmla v23.4s, bb3.4s, a1a.s[2]\n"
- "fmla v27.4s, bb3.4s, a2a.s[2]\n"
- "fmla v31.4s, bb3.4s, a3a.s[2]\n"
-
- // Unroll 7
- "fmla v16.4s, b0a.4s, a0a.s[3]\n"
- "fmla v17.4s, b1a.4s, a0a.s[3]\n"
- "fmla v18.4s, b2a.4s, a0a.s[3]\n"
- "fmla v19.4s, b3a.4s, a0a.s[3]\n"
- "cbnz %w[odds], 6f\n"
-
- "fmla v20.4s, b0a.4s, a1a.s[3]\n"
- "str q16, [%[c_ptr0]]\n"
- "fmla v21.4s, b1a.4s, a1a.s[3]\n"
- "str q17, [%[c_ptr0], #16]\n"
- "fmla v22.4s, b2a.4s, a1a.s[3]\n"
- "str q18, [%[c_ptr0], #32]\n"
- "fmla v23.4s, b3a.4s, a1a.s[3]\n"
- "str q19, [%[c_ptr0], #48]\n"
-
- "fmla v24.4s, b0a.4s, a2a.s[3]\n"
- "str q20, [%[c_ptr1]]\n"
- "fmla v25.4s, b1a.4s, a2a.s[3]\n"
- "str q21, [%[c_ptr1], #16]\n"
- "fmla v26.4s, b2a.4s, a2a.s[3]\n"
- "str q22, [%[c_ptr1], #32]\n"
- "fmla v27.4s, b3a.4s, a2a.s[3]\n"
- "str q23, [%[c_ptr1], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3a.s[3]\n"
- "str q24, [%[c_ptr2]]\n"
- "fmla v29.4s, b1a.4s, a3a.s[3]\n"
- "str q25, [%[c_ptr2], #16]\n"
- "fmla v30.4s, b2a.4s, a3a.s[3]\n"
- "str q26, [%[c_ptr2], #32]\n"
- "fmla v31.4s, b3a.4s, a3a.s[3]\n"
- "str q27, [%[c_ptr2], #48]\n"
- "b 3f\n"
-
- // Odd K case: Just do 4 more.
- "2:\n"
- "fmla v21.4s, bb1.4s, a1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #16\n"
- "fmla v25.4s, bb1.4s, a2.s[0]\n"
- "add %[a_ptr1], %[a_ptr1], #16\n"
- "fmla v29.4s, bb1.4s, a3.s[0]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[0]\n"
- "add %[a_ptr2], %[a_ptr2], #16\n"
- "fmla v22.4s, bb2.4s, a1.s[0]\n"
- "add %[a_ptr3], %[a_ptr3], #16\n"
- "fmla v26.4s, bb2.4s, a2.s[0]\n"
- "fmla v30.4s, bb2.4s, a3.s[0]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[0]\n"
- "fmla v23.4s, bb3.4s, a1.s[0]\n"
- "fmla v27.4s, bb3.4s, a2.s[0]\n"
- "fmla v31.4s, bb3.4s, a3.s[0]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- // Unroll 1
- "fmla v16.4s, b0a.4s, a0.s[1]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v20.4s, b0a.4s, a1.s[1]\n"
- "fmla v24.4s, b0a.4s, a2.s[1]\n"
- "fmla v28.4s, b0a.4s, a3.s[1]\n"
- "ldr b0aq, [%[b_ptr]]\n"
-
- "fmla v17.4s, b1a.4s, a0.s[1]\n"
- "fmla v21.4s, b1a.4s, a1.s[1]\n"
- "fmla v25.4s, b1a.4s, a2.s[1]\n"
- "fmla v29.4s, b1a.4s, a3.s[1]\n"
- "ldr b1aq, [%[b_ptr], #16]\n"
-
- "fmla v18.4s, b2a.4s, a0.s[1]\n"
- "fmla v22.4s, b2a.4s, a1.s[1]\n"
- "fmla v26.4s, b2a.4s, a2.s[1]\n"
- "fmla v30.4s, b2a.4s, a3.s[1]\n"
- "ldr b2aq, [%[b_ptr], #32]\n"
-
- "fmla v19.4s, b3a.4s, a0.s[1]\n"
- "fmla v23.4s, b3a.4s, a1.s[1]\n"
- "fmla v27.4s, b3a.4s, a2.s[1]\n"
- "fmla v31.4s, b3a.4s, a3.s[1]\n"
- "ldr b3aq, [%[b_ptr], #48]\n"
-
- // Unroll 2
- "fmla v16.4s, bb0.4s, a0.s[2]\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v20.4s, bb0.4s, a1.s[2]\n"
- "fmla v24.4s, bb0.4s, a2.s[2]\n"
- "fmla v28.4s, bb0.4s, a3.s[2]\n"
-
- "fmla v17.4s, bb1.4s, a0.s[2]\n"
- "fmla v21.4s, bb1.4s, a1.s[2]\n"
- "fmla v25.4s, bb1.4s, a2.s[2]\n"
- "fmla v29.4s, bb1.4s, a3.s[2]\n"
-
- "fmla v18.4s, bb2.4s, a0.s[2]\n"
- "fmla v22.4s, bb2.4s, a1.s[2]\n"
- "fmla v26.4s, bb2.4s, a2.s[2]\n"
- "fmla v30.4s, bb2.4s, a3.s[2]\n"
-
- "fmla v19.4s, bb3.4s, a0.s[2]\n"
- "fmla v23.4s, bb3.4s, a1.s[2]\n"
- "fmla v27.4s, bb3.4s, a2.s[2]\n"
- "fmla v31.4s, bb3.4s, a3.s[2]\n"
-
- // Unroll 3
- "fmla v16.4s, b0a.4s, a0.s[3]\n"
- "fmla v17.4s, b1a.4s, a0.s[3]\n"
- "fmla v18.4s, b2a.4s, a0.s[3]\n"
- "fmla v19.4s, b3a.4s, a0.s[3]\n"
- "cbnz %w[odds], 7f\n"
-
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "str q16, [%[c_ptr0]]\n"
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "str q17, [%[c_ptr0], #16]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- "str q18, [%[c_ptr0], #32]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "str q19, [%[c_ptr0], #48]\n"
-
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "str q20, [%[c_ptr1]]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "str q21, [%[c_ptr1], #16]\n"
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "str q22, [%[c_ptr1], #32]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "str q23, [%[c_ptr1], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "str q24, [%[c_ptr2]]\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "str q25, [%[c_ptr2], #16]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "str q26, [%[c_ptr2], #32]\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
- "str q27, [%[c_ptr2], #48]\n"
- "b 3f\n"
-
- // "Odd ones" - lead in from even
- "6:\n"
- "fmla v20.4s, b0a.4s, a1a.s[3]\n"
- "fmla v21.4s, b1a.4s, a1a.s[3]\n"
- "ldr b0q, [%[b_ptr]]\n"
- "fmla v22.4s, b2a.4s, a1a.s[3]\n"
- "subs %w[odds], %w[odds], #1\n"
- "fmla v23.4s, b3a.4s, a1a.s[3]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v24.4s, b0a.4s, a2a.s[3]\n"
- "fmla v25.4s, b1a.4s, a2a.s[3]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "fmla v26.4s, b2a.4s, a2a.s[3]\n"
- "fmla v27.4s, b3a.4s, a2a.s[3]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3a.s[3]\n"
- "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
- "fmla v29.4s, b1a.4s, a3a.s[3]\n"
- "fmla v30.4s, b2a.4s, a3a.s[3]\n"
- "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
- "fmla v31.4s, b3a.4s, a3a.s[3]\n"
-
- "fmla v16.4s, bb0.4s, a0.4s\n"
- "beq 9f\n"
- "b 8f\n"
-
- // "Odd ones" - lead in from odd
- "7:\n"
- "fmla v20.4s, b0a.4s, a1.s[3]\n"
- "subs %w[odds], %w[odds], #1\n"
- "fmla v21.4s, b1a.4s, a1.s[3]\n"
- "ldr b0q, [%[b_ptr]]\n"
- "fmla v22.4s, b2a.4s, a1.s[3]\n"
- "fmla v23.4s, b3a.4s, a1.s[3]\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v24.4s, b0a.4s, a2.s[3]\n"
- "fmla v25.4s, b1a.4s, a2.s[3]\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "fmla v26.4s, b2a.4s, a2.s[3]\n"
- "fmla v27.4s, b3a.4s, a2.s[3]\n"
- "ldr b3q, [%[b_ptr], #48]\n"
-
- "fmla v28.4s, b0a.4s, a3.s[3]\n"
- "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
- "fmla v29.4s, b1a.4s, a3.s[3]\n"
- "fmla v30.4s, b2a.4s, a3.s[3]\n"
- "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
- "fmla v31.4s, b3a.4s, a3.s[3]\n"
-
- "fmla v16.4s, bb0.4s, a0.4s\n"
- "beq 9f\n"
-
- // "Odd ones" - loop
- "8:\n"
- "fmla v17.4s, bb1.4s, a0.4s\n"
- "ld1r {a2.4s}, [%[a_ptr2]], #4\n"
- "fmla v18.4s, bb2.4s, a0.4s\n"
- "add %[b_ptr], %[b_ptr], %[ldb]\n"
- "fmla v19.4s, bb3.4s, a0.4s\n"
- "ld1r {a3.4s}, [%[a_ptr3]], #4\n"
-
- "fmla v20.4s, bb0.4s, a1.4s\n"
- "subs %w[odds], %w[odds], #1\n"
- "fmla v21.4s, bb1.4s, a1.4s\n"
- "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
- "fmla v22.4s, bb2.4s, a1.4s\n"
- "fmla v23.4s, bb3.4s, a1.4s\n"
- "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
-
- "fmla v24.4s, bb0.4s, a2.4s\n"
- "fmla v28.4s, bb0.4s, a3.4s\n"
- "ldr b0q, [%[b_ptr]]\n"
- "fmla v25.4s, bb1.4s, a2.4s\n"
- "fmla v29.4s, bb1.4s, a3.4s\n"
- "ldr b1q, [%[b_ptr], #16]\n"
-
- "fmla v26.4s, bb2.4s, a2.4s\n"
- "fmla v30.4s, bb2.4s, a3.4s\n"
- "ldr b2q, [%[b_ptr], #32]\n"
- "fmla v27.4s, bb3.4s, a2.4s\n"
- "fmla v31.4s, bb3.4s, a3.4s\n"
- "ldr b3q, [%[b_ptr], #48]\n"
- "fmla v16.4s, bb0.4s, a0.4s\n"
- "bne 8b\n"
-
- // "Odd ones" - detached final iteration
- "9:\n"
- "fmla v17.4s, bb1.4s, a0.4s\n"
- "ld1r {a2.4s}, [%[a_ptr2]], #4\n"
- "fmla v18.4s, bb2.4s, a0.4s\n"
- "fmla v19.4s, bb3.4s, a0.4s\n"
- "ld1r {a3.4s}, [%[a_ptr3]], #4\n"
-
- "fmla v20.4s, bb0.4s, a1.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmla v21.4s, bb1.4s, a1.4s\n"
- "str q17, [%[c_ptr0], #16]\n"
- "fmla v22.4s, bb2.4s, a1.4s\n"
- "str q18, [%[c_ptr0], #32]\n"
- "fmla v23.4s, bb3.4s, a1.4s\n"
- "str q19, [%[c_ptr0], #48]\n"
-
- "fmla v24.4s, bb0.4s, a2.4s\n"
- "str q20, [%[c_ptr1]]\n"
- "fmla v25.4s, bb1.4s, a2.4s\n"
- "str q21, [%[c_ptr1], #16]\n"
- "fmla v26.4s, bb2.4s, a2.4s\n"
- "str q22, [%[c_ptr1], #32]\n"
- "fmla v27.4s, bb3.4s, a2.4s\n"
- "str q23, [%[c_ptr1], #48]\n"
-
- "fmla v28.4s, bb0.4s, a3.4s\n"
- "str q24, [%[c_ptr2]]\n"
- "fmla v29.4s, bb1.4s, a3.4s\n"
- "str q25, [%[c_ptr2], #16]\n"
- "fmla v30.4s, bb2.4s, a3.4s\n"
- "str q26, [%[c_ptr2], #32]\n"
- "fmla v31.4s, bb3.4s, a3.4s\n"
- "str q27, [%[c_ptr2], #48]\n"
-
- "3:\n"
- "str q28, [%[c_ptr3]]\n"
- "add %[c_ptr0], %[c_ptr0], #64\n"
- "str q29, [%[c_ptr3], #16]\n"
- "add %[c_ptr1], %[c_ptr1], %[a_incr1], LSL #1\n"
- "str q30, [%[c_ptr3], #32]\n"
- "add %[c_ptr2], %[c_ptr2], %[a_incr2], LSL #1\n"
- "str q31, [%[c_ptr3], #48]\n"
- "add %[c_ptr3], %[c_ptr3], %[a_incr3], LSL #1\n"
-
- : [a_ptr0] "+r" (a_ptr0), [a_ptr1] "+r" (a_ptr1), [a_ptr2] "+r" (a_ptr2), [a_ptr3] "+r" (a_ptr3),
- [b_ptr] "+r" (b_ptr), [loops] "+r" (loops), [odds] "+r" (odds),
- [c_ptr0] "+r" (c_ptr0), [c_ptr1] "+r" (c_ptr1), [c_ptr2] "+r" (c_ptr2), [c_ptr3] "+r" (c_ptr3)
- : [ldb] "r" (ldbb), [oddk] "r" (oddk), [beta0] "r" (beta0), [betaptr] "r" (&beta),
- [a_incr1] "r" (a_incr1), [a_incr2] "r" (a_incr2), [a_incr3] "r" (a_incr3)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
- "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
- "cc", "memory"
- );
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
similarity index 65%
copy from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
index 06622d6..352a147 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,51 +23,61 @@
*/
#pragma once
-#ifdef __ARM_FEATURE_SVE
-
-
+#ifdef __aarch64__
namespace arm_gemm
{
// Actual kernel implementations
-void sve_smallK_fp32_mla_1VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-class smallK_fp32_mla_1VLx4
+class smallK_hybrid_fp32_mla_4x6
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+ typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
- return 4;
+ return 6;
}
static unsigned int out_width()
{
- return get_vector_length<float>() * 1;
+ return 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
+ StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_smallK_fp32_mla_1VLx4;
+ kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
- smallK_fp32_mla_1VLx4(const CPUInfo *ci)
- {
-
- }
+ smallK_hybrid_fp32_mla_4x6(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
-#endif // __ARM_FEATURE_SVE
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp
new file mode 100644
index 0000000..d8e8e52
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp
@@ -0,0 +1,4612 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include <algorithm>
+
+#include "arm_gemm.hpp"
+
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void a64_smallK_hybrid_fp32_mla_4x6(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
+ const long loops_count = iceildiv(N, (int)4) - 1;
+ const long ldab = lda * sizeof(float);
+ const long ldcb = ldc * sizeof(float);
+ float nullbias[4];
+ if (!bias) {
+ memset(nullbias, 0, (4 * sizeof(float)));
+ }
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ const float * const minptr = &minval;
+ const float * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ for (int y0=0; y0<M; y0+=6) {
+ long loops = loops_count;
+ long oob_rows = std::max(6 - (M-y0), 0);
+ const float *b_ptr0 = B;
+ const float *biasptr = bias ? bias : nullbias;
+ const uint64_t biasinc = bias ? 4*sizeof(float) : 0;
+ const float *a_ptr0 = A + (y0 * lda);
+
+ float *c_ptr0 = C + (y0 * ldc);
+
+ switch(K) {
+ case 9:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr s2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr s17, [a_ptr5]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 10:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr d2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr d5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr d8, [a_ptr2]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr d11, [a_ptr3]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr d14, [a_ptr4]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr d17, [a_ptr5]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 11:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr d2, [%[a_ptr0]], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr d17, [a_ptr5], #0x8\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ld1 {v2.s}[2], [%[a_ptr0]]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ld1 {v5.s}[2], [a_ptr1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ld1 {v8.s}[2], [a_ptr2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ld1 {v11.s}[2], [a_ptr3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "ld1 {v14.s}[2], [a_ptr4]\n"
+ "ld1 {v17.s}[2], [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 12:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]]\n"
+ "ldr q5, [a_ptr1]\n"
+ "ldr q8, [a_ptr2]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q11, [a_ptr3]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q14, [a_ptr4]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q17, [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "fmla v26.4s, v21.4s, v2.s[3]\n"
+ "fmla v27.4s, v21.4s, v5.s[3]\n"
+ "fmla v28.4s, v21.4s, v8.s[3]\n"
+ "fmla v29.4s, v21.4s, v11.s[3]\n"
+ "fmla v30.4s, v21.4s, v14.s[3]\n"
+ "fmla v31.4s, v21.4s, v17.s[3]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "fmla v26.4s, v21.4s, v2.s[3]\n"
+ "fmla v27.4s, v21.4s, v5.s[3]\n"
+ "fmla v28.4s, v21.4s, v8.s[3]\n"
+ "fmla v29.4s, v21.4s, v11.s[3]\n"
+ "fmla v30.4s, v21.4s, v14.s[3]\n"
+ "fmla v31.4s, v21.4s, v17.s[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "fmla v26.4s, v21.4s, v2.s[3]\n"
+ "fmla v27.4s, v21.4s, v5.s[3]\n"
+ "fmla v28.4s, v21.4s, v8.s[3]\n"
+ "fmla v29.4s, v21.4s, v11.s[3]\n"
+ "fmla v30.4s, v21.4s, v14.s[3]\n"
+ "fmla v31.4s, v21.4s, v17.s[3]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v18.4s, v0.s[0]\n"
+ "fmla v27.4s, v18.4s, v3.s[0]\n"
+ "fmla v28.4s, v18.4s, v6.s[0]\n"
+ "fmla v29.4s, v18.4s, v9.s[0]\n"
+ "fmla v30.4s, v18.4s, v12.s[0]\n"
+ "fmla v31.4s, v18.4s, v15.s[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "fmla v26.4s, v19.4s, v0.s[1]\n"
+ "fmla v27.4s, v19.4s, v3.s[1]\n"
+ "fmla v28.4s, v19.4s, v6.s[1]\n"
+ "fmla v29.4s, v19.4s, v9.s[1]\n"
+ "fmla v30.4s, v19.4s, v12.s[1]\n"
+ "fmla v31.4s, v19.4s, v15.s[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v20.4s, v0.s[2]\n"
+ "fmla v27.4s, v20.4s, v3.s[2]\n"
+ "fmla v28.4s, v20.4s, v6.s[2]\n"
+ "fmla v29.4s, v20.4s, v9.s[2]\n"
+ "fmla v30.4s, v20.4s, v12.s[2]\n"
+ "fmla v31.4s, v20.4s, v15.s[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "fmla v27.4s, v21.4s, v3.s[3]\n"
+ "fmla v28.4s, v21.4s, v6.s[3]\n"
+ "fmla v29.4s, v21.4s, v9.s[3]\n"
+ "fmla v30.4s, v21.4s, v12.s[3]\n"
+ "fmla v31.4s, v21.4s, v15.s[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "fmla v26.4s, v22.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "fmla v27.4s, v22.4s, v4.s[0]\n"
+ "fmla v28.4s, v22.4s, v7.s[0]\n"
+ "fmla v29.4s, v22.4s, v10.s[0]\n"
+ "fmla v30.4s, v22.4s, v13.s[0]\n"
+ "fmla v31.4s, v22.4s, v16.s[0]\n"
+ "fmla v26.4s, v23.4s, v1.s[1]\n"
+ "fmla v27.4s, v23.4s, v4.s[1]\n"
+ "fmla v28.4s, v23.4s, v7.s[1]\n"
+ "fmla v29.4s, v23.4s, v10.s[1]\n"
+ "fmla v30.4s, v23.4s, v13.s[1]\n"
+ "fmla v31.4s, v23.4s, v16.s[1]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v7.s[2]\n"
+ "fmla v29.4s, v24.4s, v10.s[2]\n"
+ "fmla v30.4s, v24.4s, v13.s[2]\n"
+ "fmla v31.4s, v24.4s, v16.s[2]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v7.s[3]\n"
+ "fmla v29.4s, v25.4s, v10.s[3]\n"
+ "fmla v30.4s, v25.4s, v13.s[3]\n"
+ "fmla v31.4s, v25.4s, v16.s[3]\n"
+ "fmla v26.4s, v18.4s, v2.s[0]\n"
+ "fmla v27.4s, v18.4s, v5.s[0]\n"
+ "fmla v28.4s, v18.4s, v8.s[0]\n"
+ "fmla v29.4s, v18.4s, v11.s[0]\n"
+ "fmla v30.4s, v18.4s, v14.s[0]\n"
+ "fmla v31.4s, v18.4s, v17.s[0]\n"
+ "fmla v26.4s, v19.4s, v2.s[1]\n"
+ "fmla v27.4s, v19.4s, v5.s[1]\n"
+ "fmla v28.4s, v19.4s, v8.s[1]\n"
+ "fmla v29.4s, v19.4s, v11.s[1]\n"
+ "fmla v30.4s, v19.4s, v14.s[1]\n"
+ "fmla v31.4s, v19.4s, v17.s[1]\n"
+ "fmla v26.4s, v20.4s, v2.s[2]\n"
+ "fmla v27.4s, v20.4s, v5.s[2]\n"
+ "fmla v28.4s, v20.4s, v8.s[2]\n"
+ "fmla v29.4s, v20.4s, v11.s[2]\n"
+ "fmla v30.4s, v20.4s, v14.s[2]\n"
+ "fmla v31.4s, v20.4s, v17.s[2]\n"
+ "fmla v26.4s, v21.4s, v2.s[3]\n"
+ "fmla v27.4s, v21.4s, v5.s[3]\n"
+ "fmla v28.4s, v21.4s, v8.s[3]\n"
+ "fmla v29.4s, v21.4s, v11.s[3]\n"
+ "fmla v30.4s, v21.4s, v14.s[3]\n"
+ "fmla v31.4s, v21.4s, v17.s[3]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 13:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
+ "ldr s3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr s7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr s11, [a_ptr2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr s15, [a_ptr3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr s19, [a_ptr4]\n"
+ "ldr s23, [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 14:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
+ "ldr d3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr d7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr d11, [a_ptr2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr d15, [a_ptr3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr d19, [a_ptr4]\n"
+ "ldr d23, [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 15:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
+ "ldr d3, [%[a_ptr0]], #0x8\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr d7, [a_ptr1], #0x8\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr d11, [a_ptr2], #0x8\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr d15, [a_ptr3], #0x8\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr d19, [a_ptr4], #0x8\n"
+ "ldr d23, [a_ptr5], #0x8\n"
+ "ld1 {v3.s}[2], [%[a_ptr0]]\n"
+ "ld1 {v7.s}[2], [a_ptr1]\n"
+ "ld1 {v11.s}[2], [a_ptr2]\n"
+ "ld1 {v15.s}[2], [a_ptr3]\n"
+ "ld1 {v19.s}[2], [a_ptr4]\n"
+ "ld1 {v23.s}[2], [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ default:
+ case 16:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "c_ptr1 .req X5\n"
+ "c_ptr2 .req X6\n"
+ "c_ptr3 .req X7\n"
+ "c_ptr4 .req X8\n"
+ "c_ptr5 .req X9\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr q3, [%[a_ptr0]]\n"
+ "ldr q7, [a_ptr1]\n"
+ "ldr q11, [a_ptr2]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q15, [a_ptr3]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q19, [a_ptr4]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q23, [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 2f\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "fmla v26.4s, v25.4s, v3.s[3]\n"
+ "fmla v27.4s, v25.4s, v7.s[3]\n"
+ "fmla v28.4s, v25.4s, v11.s[3]\n"
+ "fmla v29.4s, v25.4s, v15.s[3]\n"
+ "fmla v30.4s, v25.4s, v19.s[3]\n"
+ "fmla v31.4s, v25.4s, v23.s[3]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "fmla v26.4s, v25.4s, v3.s[3]\n"
+ "fmla v27.4s, v25.4s, v7.s[3]\n"
+ "fmla v28.4s, v25.4s, v11.s[3]\n"
+ "fmla v29.4s, v25.4s, v15.s[3]\n"
+ "fmla v30.4s, v25.4s, v19.s[3]\n"
+ "fmla v31.4s, v25.4s, v23.s[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "ldr q26, [%[biasptr]]\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "mov v27.16b, v26.16b\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "mov v28.16b, v26.16b\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v29.16b, v26.16b\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "str q30, [c_ptr4]\n"
+ "mov v30.16b, v26.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "str q31, [c_ptr5]\n"
+ "mov v31.16b, v26.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "fmla v26.4s, v25.4s, v3.s[3]\n"
+ "fmla v27.4s, v25.4s, v7.s[3]\n"
+ "fmla v28.4s, v25.4s, v11.s[3]\n"
+ "fmla v29.4s, v25.4s, v15.s[3]\n"
+ "fmla v30.4s, v25.4s, v19.s[3]\n"
+ "fmla v31.4s, v25.4s, v23.s[3]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q26, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v27.16b, v26.16b\n"
+ "mov v28.16b, v26.16b\n"
+ "mov v29.16b, v26.16b\n"
+ "mov v30.16b, v26.16b\n"
+ "mov v31.16b, v26.16b\n"
+ "fmla v26.4s, v24.4s, v0.s[0]\n"
+ "fmla v27.4s, v24.4s, v4.s[0]\n"
+ "fmla v28.4s, v24.4s, v8.s[0]\n"
+ "fmla v29.4s, v24.4s, v12.s[0]\n"
+ "fmla v30.4s, v24.4s, v16.s[0]\n"
+ "fmla v31.4s, v24.4s, v20.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[1]\n"
+ "fmla v27.4s, v25.4s, v4.s[1]\n"
+ "fmla v28.4s, v25.4s, v8.s[1]\n"
+ "fmla v29.4s, v25.4s, v12.s[1]\n"
+ "fmla v30.4s, v25.4s, v16.s[1]\n"
+ "fmla v31.4s, v25.4s, v20.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v4.s[2]\n"
+ "fmla v28.4s, v24.4s, v8.s[2]\n"
+ "fmla v29.4s, v24.4s, v12.s[2]\n"
+ "fmla v30.4s, v24.4s, v16.s[2]\n"
+ "fmla v31.4s, v24.4s, v20.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v0.s[3]\n"
+ "fmla v27.4s, v25.4s, v4.s[3]\n"
+ "fmla v28.4s, v25.4s, v8.s[3]\n"
+ "fmla v29.4s, v25.4s, v12.s[3]\n"
+ "fmla v30.4s, v25.4s, v16.s[3]\n"
+ "fmla v31.4s, v25.4s, v20.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[0]\n"
+ "fmla v28.4s, v24.4s, v9.s[0]\n"
+ "fmla v29.4s, v24.4s, v13.s[0]\n"
+ "fmla v30.4s, v24.4s, v17.s[0]\n"
+ "fmla v31.4s, v24.4s, v21.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[1]\n"
+ "fmla v27.4s, v25.4s, v5.s[1]\n"
+ "fmla v28.4s, v25.4s, v9.s[1]\n"
+ "fmla v29.4s, v25.4s, v13.s[1]\n"
+ "fmla v30.4s, v25.4s, v17.s[1]\n"
+ "fmla v31.4s, v25.4s, v21.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v5.s[2]\n"
+ "fmla v28.4s, v24.4s, v9.s[2]\n"
+ "fmla v29.4s, v24.4s, v13.s[2]\n"
+ "fmla v30.4s, v24.4s, v17.s[2]\n"
+ "fmla v31.4s, v24.4s, v21.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v1.s[3]\n"
+ "fmla v27.4s, v25.4s, v5.s[3]\n"
+ "fmla v28.4s, v25.4s, v9.s[3]\n"
+ "fmla v29.4s, v25.4s, v13.s[3]\n"
+ "fmla v30.4s, v25.4s, v17.s[3]\n"
+ "fmla v31.4s, v25.4s, v21.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[0]\n"
+ "fmla v28.4s, v24.4s, v10.s[0]\n"
+ "fmla v29.4s, v24.4s, v14.s[0]\n"
+ "fmla v30.4s, v24.4s, v18.s[0]\n"
+ "fmla v31.4s, v24.4s, v22.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[1]\n"
+ "fmla v27.4s, v25.4s, v6.s[1]\n"
+ "fmla v28.4s, v25.4s, v10.s[1]\n"
+ "fmla v29.4s, v25.4s, v14.s[1]\n"
+ "fmla v30.4s, v25.4s, v18.s[1]\n"
+ "fmla v31.4s, v25.4s, v22.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v2.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v6.s[2]\n"
+ "fmla v28.4s, v24.4s, v10.s[2]\n"
+ "fmla v29.4s, v24.4s, v14.s[2]\n"
+ "fmla v30.4s, v24.4s, v18.s[2]\n"
+ "fmla v31.4s, v24.4s, v22.s[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v2.s[3]\n"
+ "fmla v27.4s, v25.4s, v6.s[3]\n"
+ "fmla v28.4s, v25.4s, v10.s[3]\n"
+ "fmla v29.4s, v25.4s, v14.s[3]\n"
+ "fmla v30.4s, v25.4s, v18.s[3]\n"
+ "fmla v31.4s, v25.4s, v22.s[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[0]\n"
+ "fmla v28.4s, v24.4s, v11.s[0]\n"
+ "fmla v29.4s, v24.4s, v15.s[0]\n"
+ "fmla v30.4s, v24.4s, v19.s[0]\n"
+ "fmla v31.4s, v24.4s, v23.s[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "fmla v26.4s, v25.4s, v3.s[1]\n"
+ "fmla v27.4s, v25.4s, v7.s[1]\n"
+ "fmla v28.4s, v25.4s, v11.s[1]\n"
+ "fmla v29.4s, v25.4s, v15.s[1]\n"
+ "fmla v30.4s, v25.4s, v19.s[1]\n"
+ "fmla v31.4s, v25.4s, v23.s[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "fmla v26.4s, v24.4s, v3.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmla v27.4s, v24.4s, v7.s[2]\n"
+ "fmla v28.4s, v24.4s, v11.s[2]\n"
+ "fmla v29.4s, v24.4s, v15.s[2]\n"
+ "fmla v30.4s, v24.4s, v19.s[2]\n"
+ "fmla v31.4s, v24.4s, v23.s[2]\n"
+ "fmla v26.4s, v25.4s, v3.s[3]\n"
+ "fmla v27.4s, v25.4s, v7.s[3]\n"
+ "fmla v28.4s, v25.4s, v11.s[3]\n"
+ "fmla v29.4s, v25.4s, v15.s[3]\n"
+ "fmla v30.4s, v25.4s, v19.s[3]\n"
+ "fmla v31.4s, v25.4s, v23.s[3]\n"
+ "5:\n"
+ "ld1r {v24.4s}, [%[minptr]]\n"
+ "ld1r {v25.4s}, [%[maxptr]]\n"
+ "fmax v26.4s, v26.4s, v24.4s\n"
+ "fmax v27.4s, v27.4s, v24.4s\n"
+ "fmax v28.4s, v28.4s, v24.4s\n"
+ "fmax v29.4s, v29.4s, v24.4s\n"
+ "fmin v26.4s, v26.4s, v25.4s\n"
+ "fmin v27.4s, v27.4s, v25.4s\n"
+ "fmin v28.4s, v28.4s, v25.4s\n"
+ "fmin v29.4s, v29.4s, v25.4s\n"
+ "str q26, [%[c_ptr0]]\n"
+ "fmax v30.4s, v30.4s, v24.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v31.4s, v31.4s, v24.4s\n"
+ "str q27, [c_ptr1]\n"
+ "fmin v30.4s, v30.4s, v25.4s\n"
+ "fmin v31.4s, v31.4s, v25.4s\n"
+ "str q28, [c_ptr2]\n"
+ "str q29, [c_ptr3]\n"
+ "str q30, [c_ptr4]\n"
+ "str q31, [c_ptr5]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
similarity index 65%
copy from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
index 06622d6..c5d39cb 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,7 +23,7 @@
*/
#pragma once
-#ifdef __ARM_FEATURE_SVE
+#ifdef __aarch64__
@@ -31,43 +31,55 @@
{
// Actual kernel implementations
-void sve_smallK_fp32_mla_1VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+void a64_smallK_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-class smallK_fp32_mla_1VLx4
+class smallK_hybrid_fp32_mla_4x8
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+ typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
- return 4;
+ return 8;
}
static unsigned int out_width()
{
- return get_vector_length<float>() * 1;
+ return 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
+ StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_smallK_fp32_mla_1VLx4;
+ kern_type kernel=a64_smallK_hybrid_fp32_mla_4x8;
- smallK_fp32_mla_1VLx4(const CPUInfo *ci)
- {
-
- }
+ smallK_hybrid_fp32_mla_4x8(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
-#endif // __ARM_FEATURE_SVE
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp
new file mode 100644
index 0000000..7ad5205
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp
@@ -0,0 +1,3340 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include <algorithm>
+
+#include "arm_gemm.hpp"
+
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void a64_smallK_hybrid_fp32_mla_4x8(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
+ const long loops_count = iceildiv(N, (int)4) - 1;
+ const long ldab = lda * sizeof(float);
+ const long ldcb = ldc * sizeof(float);
+ float nullbias[4];
+ if (!bias) {
+ memset(nullbias, 0, (4 * sizeof(float)));
+ }
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ const float * const minptr = &minval;
+ const float * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ for (int y0=0; y0<M; y0+=8) {
+ long loops = loops_count;
+ long oob_rows = std::max(8 - (M-y0), 0);
+ const float *b_ptr0 = B;
+ const float *biasptr = bias ? bias : nullbias;
+ const uint64_t biasinc = bias ? 4*sizeof(float) : 0;
+ const float *a_ptr0 = A + (y0 * lda);
+
+ float *c_ptr0 = C + (y0 * ldc);
+
+ switch(K) {
+ case 1:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr s0, [%[a_ptr0]]\n"
+ "ldr s1, [a_ptr1]\n"
+ "ldr s2, [a_ptr2]\n"
+ "ldr s3, [a_ptr3]\n"
+ "ldr s4, [a_ptr4]\n"
+ "ldr s5, [a_ptr5]\n"
+ "ldr s6, [a_ptr6]\n"
+ "ldr s7, [a_ptr7]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v27.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v28.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v29.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v27.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v28.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v29.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr d0, [%[a_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "ldr d1, [a_ptr1]\n"
+ "ldr d2, [a_ptr2]\n"
+ "ldr d3, [a_ptr3]\n"
+ "ldr d4, [a_ptr4]\n"
+ "ldr d5, [a_ptr5]\n"
+ "ldr d6, [a_ptr6]\n"
+ "ldr d7, [a_ptr7]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "mov v30.16b, v24.16b\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v27.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v28.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v29.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr d0, [%[a_ptr0]], #0x8\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
+ "ld1 {v0.s}[2], [%[a_ptr0]]\n"
+ "ld1 {v1.s}[2], [a_ptr1]\n"
+ "ld1 {v2.s}[2], [a_ptr2]\n"
+ "ld1 {v3.s}[2], [a_ptr3]\n"
+ "ld1 {v4.s}[2], [a_ptr4]\n"
+ "ld1 {v5.s}[2], [a_ptr5]\n"
+ "ld1 {v6.s}[2], [a_ptr6]\n"
+ "ld1 {v7.s}[2], [a_ptr7]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "mov v30.16b, v24.16b\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "mov v30.16b, v24.16b\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "ldr q1, [a_ptr1]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "ldr q2, [a_ptr2]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "ldr q3, [a_ptr3]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "ldr q4, [a_ptr4]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "ldr q5, [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "ldr q6, [a_ptr6]\n"
+ "ldr q7, [a_ptr7]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v1.s[3]\n"
+ "fmla v26.4s, v19.4s, v2.s[3]\n"
+ "fmla v27.4s, v19.4s, v3.s[3]\n"
+ "fmla v28.4s, v19.4s, v4.s[3]\n"
+ "fmla v29.4s, v19.4s, v5.s[3]\n"
+ "fmla v30.4s, v19.4s, v6.s[3]\n"
+ "fmla v31.4s, v19.4s, v7.s[3]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v27.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v28.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v29.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v1.s[3]\n"
+ "fmla v26.4s, v19.4s, v2.s[3]\n"
+ "fmla v27.4s, v19.4s, v3.s[3]\n"
+ "fmla v28.4s, v19.4s, v4.s[3]\n"
+ "fmla v29.4s, v19.4s, v5.s[3]\n"
+ "fmla v30.4s, v19.4s, v6.s[3]\n"
+ "fmla v31.4s, v19.4s, v7.s[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "mov v30.16b, v24.16b\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v1.s[3]\n"
+ "fmla v26.4s, v19.4s, v2.s[3]\n"
+ "fmla v27.4s, v19.4s, v3.s[3]\n"
+ "fmla v28.4s, v19.4s, v4.s[3]\n"
+ "fmla v29.4s, v19.4s, v5.s[3]\n"
+ "fmla v30.4s, v19.4s, v6.s[3]\n"
+ "fmla v31.4s, v19.4s, v7.s[3]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v1.s[1]\n"
+ "fmla v26.4s, v17.4s, v2.s[1]\n"
+ "fmla v27.4s, v17.4s, v3.s[1]\n"
+ "fmla v28.4s, v17.4s, v4.s[1]\n"
+ "fmla v29.4s, v17.4s, v5.s[1]\n"
+ "fmla v30.4s, v17.4s, v6.s[1]\n"
+ "fmla v31.4s, v17.4s, v7.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v1.s[2]\n"
+ "fmla v26.4s, v18.4s, v2.s[2]\n"
+ "fmla v27.4s, v18.4s, v3.s[2]\n"
+ "fmla v28.4s, v18.4s, v4.s[2]\n"
+ "fmla v29.4s, v18.4s, v5.s[2]\n"
+ "fmla v30.4s, v18.4s, v6.s[2]\n"
+ "fmla v31.4s, v18.4s, v7.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v1.s[3]\n"
+ "fmla v26.4s, v19.4s, v2.s[3]\n"
+ "fmla v27.4s, v19.4s, v3.s[3]\n"
+ "fmla v28.4s, v19.4s, v4.s[3]\n"
+ "fmla v29.4s, v19.4s, v5.s[3]\n"
+ "fmla v30.4s, v19.4s, v6.s[3]\n"
+ "fmla v31.4s, v19.4s, v7.s[3]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 5:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
+ "ldr s1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr s3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr s5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr s7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr s9, [a_ptr4]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr s11, [a_ptr5]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "ldr s13, [a_ptr6]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "ldr s15, [a_ptr7]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v25.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "mov v26.16b, v24.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q29, [c_ptr5]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "mov v30.16b, v24.16b\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v27.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v28.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v29.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 6:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
+ "ldr d1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr d3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr d5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr d7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr d9, [a_ptr4]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr d11, [a_ptr5]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "ldr d13, [a_ptr6]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "ldr d15, [a_ptr7]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v25.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "mov v26.16b, v24.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q29, [c_ptr5]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "mov v30.16b, v24.16b\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "mov v25.16b, v24.16b\n"
+ "str q28, [c_ptr4]\n"
+ "mov v26.16b, v24.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q29, [c_ptr5]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q30, [c_ptr6]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "mov v30.16b, v24.16b\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ case 7:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
+ "ldr d1, [%[a_ptr0]], #0x8\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr d3, [a_ptr1], #0x8\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr d5, [a_ptr2], #0x8\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr d7, [a_ptr3], #0x8\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr d9, [a_ptr4], #0x8\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr d11, [a_ptr5], #0x8\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "ldr d13, [a_ptr6], #0x8\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "ldr d15, [a_ptr7], #0x8\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ld1 {v1.s}[2], [%[a_ptr0]]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "ld1 {v3.s}[2], [a_ptr1]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "ld1 {v5.s}[2], [a_ptr2]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "ld1 {v7.s}[2], [a_ptr3]\n"
+ "ld1 {v9.s}[2], [a_ptr4]\n"
+ "ld1 {v11.s}[2], [a_ptr5]\n"
+ "ld1 {v13.s}[2], [a_ptr6]\n"
+ "ld1 {v15.s}[2], [a_ptr7]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "mov v26.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "mov v26.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "mov v27.16b, v24.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "mov v28.16b, v24.16b\n"
+ "str q29, [c_ptr5]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ default:
+ case 8:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "a_ptr4 .req X3\n"
+ "a_ptr5 .req X4\n"
+ "a_ptr6 .req X5\n"
+ "a_ptr7 .req X6\n"
+ "c_ptr1 .req X7\n"
+ "c_ptr2 .req X8\n"
+ "c_ptr3 .req X9\n"
+ "c_ptr4 .req X10\n"
+ "c_ptr5 .req X11\n"
+ "c_ptr6 .req X12\n"
+ "c_ptr7 .req X13\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "add a_ptr4, a_ptr3, %[lda]\n"
+ "add c_ptr4, c_ptr3, %[ldc]\n"
+ "add a_ptr5, a_ptr4, %[lda]\n"
+ "add c_ptr5, c_ptr4, %[ldc]\n"
+ "add a_ptr6, a_ptr5, %[lda]\n"
+ "add c_ptr6, c_ptr5, %[ldc]\n"
+ "add a_ptr7, a_ptr6, %[lda]\n"
+ "add c_ptr7, c_ptr6, %[ldc]\n"
+ "cbz %[oob_rows], 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr7, %[c_ptr0], #0x0\n"
+ "add a_ptr7, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr6, %[c_ptr0], #0x0\n"
+ "add a_ptr6, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr5, %[c_ptr0], #0x0\n"
+ "add a_ptr5, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr4, %[c_ptr0], #0x0\n"
+ "add a_ptr4, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr3, %[c_ptr0], #0x0\n"
+ "add a_ptr3, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr2, %[c_ptr0], #0x0\n"
+ "add a_ptr2, %[a_ptr0], #0x0\n"
+ "b.eq 1f\n"
+ "subs %[oob_rows], %[oob_rows], #0x1\n"
+ "add c_ptr1, %[c_ptr0], #0x0\n"
+ "add a_ptr1, %[a_ptr0], #0x0\n"
+ "1:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr q1, [%[a_ptr0]]\n"
+ "ldr q3, [a_ptr1]\n"
+ "ldr q5, [a_ptr2]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
+ "ldr q7, [a_ptr3]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "ldr q9, [a_ptr4]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "ldr q11, [a_ptr5]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "ldr q13, [a_ptr6]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "ldr q15, [a_ptr7]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 2f\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "fmla v24.4s, v23.4s, v1.s[3]\n"
+ "fmla v25.4s, v23.4s, v3.s[3]\n"
+ "fmla v26.4s, v23.4s, v5.s[3]\n"
+ "fmla v27.4s, v23.4s, v7.s[3]\n"
+ "fmla v28.4s, v23.4s, v9.s[3]\n"
+ "fmla v29.4s, v23.4s, v11.s[3]\n"
+ "fmla v30.4s, v23.4s, v13.s[3]\n"
+ "fmla v31.4s, v23.4s, v15.s[3]\n"
+ "b.eq 3f\n"
+ "4:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "mov v26.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "mov v27.16b, v24.16b\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v24.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "str q29, [c_ptr5]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "fmla v24.4s, v23.4s, v1.s[3]\n"
+ "fmla v25.4s, v23.4s, v3.s[3]\n"
+ "fmla v26.4s, v23.4s, v5.s[3]\n"
+ "fmla v27.4s, v23.4s, v7.s[3]\n"
+ "fmla v28.4s, v23.4s, v9.s[3]\n"
+ "fmla v29.4s, v23.4s, v11.s[3]\n"
+ "fmla v30.4s, v23.4s, v13.s[3]\n"
+ "fmla v31.4s, v23.4s, v15.s[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "ldr q24, [%[biasptr]]\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q27, [c_ptr3]\n"
+ "mov v25.16b, v24.16b\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "mov v26.16b, v24.16b\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q28, [c_ptr4]\n"
+ "mov v27.16b, v24.16b\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
+ "mov v28.16b, v24.16b\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "str q29, [c_ptr5]\n"
+ "mov v29.16b, v24.16b\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "str q30, [c_ptr6]\n"
+ "mov v30.16b, v24.16b\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "str q31, [c_ptr7]\n"
+ "mov v31.16b, v24.16b\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "fmla v24.4s, v23.4s, v1.s[3]\n"
+ "fmla v25.4s, v23.4s, v3.s[3]\n"
+ "fmla v26.4s, v23.4s, v5.s[3]\n"
+ "fmla v27.4s, v23.4s, v7.s[3]\n"
+ "fmla v28.4s, v23.4s, v9.s[3]\n"
+ "fmla v29.4s, v23.4s, v11.s[3]\n"
+ "fmla v30.4s, v23.4s, v13.s[3]\n"
+ "fmla v31.4s, v23.4s, v15.s[3]\n"
+ "b 5f\n"
+ "2:\n"
+ "ldr q24, [%[biasptr]]\n"
+ "add %[biasptr], %[biasptr], %[biasinc]\n"
+ "mov v25.16b, v24.16b\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "fmla v25.4s, v16.4s, v2.s[0]\n"
+ "fmla v26.4s, v16.4s, v4.s[0]\n"
+ "fmla v27.4s, v16.4s, v6.s[0]\n"
+ "fmla v28.4s, v16.4s, v8.s[0]\n"
+ "fmla v29.4s, v16.4s, v10.s[0]\n"
+ "fmla v30.4s, v16.4s, v12.s[0]\n"
+ "fmla v31.4s, v16.4s, v14.s[0]\n"
+ "fmla v24.4s, v17.4s, v0.s[1]\n"
+ "fmla v25.4s, v17.4s, v2.s[1]\n"
+ "fmla v26.4s, v17.4s, v4.s[1]\n"
+ "fmla v27.4s, v17.4s, v6.s[1]\n"
+ "fmla v28.4s, v17.4s, v8.s[1]\n"
+ "fmla v29.4s, v17.4s, v10.s[1]\n"
+ "fmla v30.4s, v17.4s, v12.s[1]\n"
+ "fmla v31.4s, v17.4s, v14.s[1]\n"
+ "fmla v24.4s, v18.4s, v0.s[2]\n"
+ "fmla v25.4s, v18.4s, v2.s[2]\n"
+ "fmla v26.4s, v18.4s, v4.s[2]\n"
+ "fmla v27.4s, v18.4s, v6.s[2]\n"
+ "fmla v28.4s, v18.4s, v8.s[2]\n"
+ "fmla v29.4s, v18.4s, v10.s[2]\n"
+ "fmla v30.4s, v18.4s, v12.s[2]\n"
+ "fmla v31.4s, v18.4s, v14.s[2]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "fmla v25.4s, v19.4s, v2.s[3]\n"
+ "fmla v26.4s, v19.4s, v4.s[3]\n"
+ "fmla v27.4s, v19.4s, v6.s[3]\n"
+ "fmla v28.4s, v19.4s, v8.s[3]\n"
+ "fmla v29.4s, v19.4s, v10.s[3]\n"
+ "fmla v30.4s, v19.4s, v12.s[3]\n"
+ "fmla v31.4s, v19.4s, v14.s[3]\n"
+ "fmla v24.4s, v20.4s, v1.s[0]\n"
+ "fmla v25.4s, v20.4s, v3.s[0]\n"
+ "fmla v26.4s, v20.4s, v5.s[0]\n"
+ "fmla v27.4s, v20.4s, v7.s[0]\n"
+ "fmla v28.4s, v20.4s, v9.s[0]\n"
+ "fmla v29.4s, v20.4s, v11.s[0]\n"
+ "fmla v30.4s, v20.4s, v13.s[0]\n"
+ "fmla v31.4s, v20.4s, v15.s[0]\n"
+ "fmla v24.4s, v21.4s, v1.s[1]\n"
+ "fmla v25.4s, v21.4s, v3.s[1]\n"
+ "fmla v26.4s, v21.4s, v5.s[1]\n"
+ "fmla v27.4s, v21.4s, v7.s[1]\n"
+ "fmla v28.4s, v21.4s, v9.s[1]\n"
+ "fmla v29.4s, v21.4s, v11.s[1]\n"
+ "fmla v30.4s, v21.4s, v13.s[1]\n"
+ "fmla v31.4s, v21.4s, v15.s[1]\n"
+ "fmla v24.4s, v22.4s, v1.s[2]\n"
+ "fmla v25.4s, v22.4s, v3.s[2]\n"
+ "fmla v26.4s, v22.4s, v5.s[2]\n"
+ "fmla v27.4s, v22.4s, v7.s[2]\n"
+ "fmla v28.4s, v22.4s, v9.s[2]\n"
+ "fmla v29.4s, v22.4s, v11.s[2]\n"
+ "fmla v30.4s, v22.4s, v13.s[2]\n"
+ "fmla v31.4s, v22.4s, v15.s[2]\n"
+ "fmla v24.4s, v23.4s, v1.s[3]\n"
+ "fmla v25.4s, v23.4s, v3.s[3]\n"
+ "fmla v26.4s, v23.4s, v5.s[3]\n"
+ "fmla v27.4s, v23.4s, v7.s[3]\n"
+ "fmla v28.4s, v23.4s, v9.s[3]\n"
+ "fmla v29.4s, v23.4s, v11.s[3]\n"
+ "fmla v30.4s, v23.4s, v13.s[3]\n"
+ "fmla v31.4s, v23.4s, v15.s[3]\n"
+ "5:\n"
+ "ld1r {v22.4s}, [%[minptr]]\n"
+ "ld1r {v23.4s}, [%[maxptr]]\n"
+ "fmax v24.4s, v24.4s, v22.4s\n"
+ "fmax v25.4s, v25.4s, v22.4s\n"
+ "fmax v26.4s, v26.4s, v22.4s\n"
+ "fmax v27.4s, v27.4s, v22.4s\n"
+ "fmin v24.4s, v24.4s, v23.4s\n"
+ "fmin v25.4s, v25.4s, v23.4s\n"
+ "fmin v26.4s, v26.4s, v23.4s\n"
+ "fmin v27.4s, v27.4s, v23.4s\n"
+ "str q24, [%[c_ptr0]]\n"
+ "fmax v28.4s, v28.4s, v22.4s\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "fmax v29.4s, v29.4s, v22.4s\n"
+ "str q25, [c_ptr1]\n"
+ "fmax v30.4s, v30.4s, v22.4s\n"
+ "fmin v28.4s, v28.4s, v23.4s\n"
+ "fmax v31.4s, v31.4s, v22.4s\n"
+ "str q26, [c_ptr2]\n"
+ "fmin v29.4s, v29.4s, v23.4s\n"
+ "fmin v30.4s, v30.4s, v23.4s\n"
+ "fmin v31.4s, v31.4s, v23.4s\n"
+ "str q27, [c_ptr3]\n"
+ "str q28, [c_ptr4]\n"
+ "str q29, [c_ptr5]\n"
+ "str q30, [c_ptr6]\n"
+ "str q31, [c_ptr7]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq a_ptr4\n"
+ ".unreq a_ptr5\n"
+ ".unreq a_ptr6\n"
+ ".unreq a_ptr7\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ ".unreq c_ptr4\n"
+ ".unreq c_ptr5\n"
+ ".unreq c_ptr6\n"
+ ".unreq c_ptr7\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr)
+ : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ break;
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp
index 7fa02e3..a1a1fc5 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 ARM Limited.
+ * Copyright (c) 2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -31,8 +31,8 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
-void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
class smallK_hybrid_s8s32_dot_4x6
{
@@ -40,10 +40,10 @@
typedef int8_t operand_type;
typedef int32_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+ typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 6;
}
@@ -53,11 +53,26 @@
return 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {};
// Default to the generic kernel
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp
index e8ac2d7..6d71840 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp
@@ -25,15 +25,16 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- UNUSED(beta);
- const long loops_count = (N / 4) - 1;
+void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+ const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
const long odds_count = K % 4;
@@ -97,335 +98,356 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr s2, [%[a_ptr0]]\n"
- "ldr s6, [a_ptr1]\n"
- "ldr s10, [a_ptr2]\n"
- "ldr s14, [a_ptr3]\n"
- "ldr s18, [a_ptr4]\n"
- "ldr s22, [a_ptr5]\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
- "ldr b6, [a_ptr1]\n"
- "ldr b10, [a_ptr2]\n"
- "ldr b14, [a_ptr3]\n"
- "ldr b18, [a_ptr4]\n"
- "ldr b22, [a_ptr5]\n"
+ "ldr b5, [a_ptr1]\n"
+ "ldr b8, [a_ptr2]\n"
+ "ldr b11, [a_ptr3]\n"
+ "ldr b14, [a_ptr4]\n"
+ "ldr b17, [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ldr h2, [%[a_ptr0]], #0x2\n"
- "ldr h6, [a_ptr1], #0x2\n"
- "ldr h10, [a_ptr2], #0x2\n"
- "ldr h14, [a_ptr3], #0x2\n"
- "ldr h18, [a_ptr4], #0x2\n"
- "ldr h22, [a_ptr5], #0x2\n"
+ "ldr h5, [a_ptr1], #0x2\n"
+ "ldr h8, [a_ptr2], #0x2\n"
+ "ldr h11, [a_ptr3], #0x2\n"
+ "ldr h14, [a_ptr4], #0x2\n"
+ "ldr h17, [a_ptr5], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[2], [%[a_ptr0]]\n"
- "ld1 {v6.b}[2], [a_ptr1]\n"
- "ld1 {v10.b}[2], [a_ptr2]\n"
- "ld1 {v14.b}[2], [a_ptr3]\n"
- "ld1 {v18.b}[2], [a_ptr4]\n"
- "ld1 {v22.b}[2], [a_ptr5]\n"
+ "ld1 {v5.b}[2], [a_ptr1]\n"
+ "ld1 {v8.b}[2], [a_ptr2]\n"
+ "ld1 {v11.b}[2], [a_ptr3]\n"
+ "ld1 {v14.b}[2], [a_ptr4]\n"
+ "ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
"ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -493,364 +515,385 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
- "ldr d6, [a_ptr1]\n"
- "ldr d10, [a_ptr2]\n"
- "ldr d14, [a_ptr3]\n"
- "ldr d18, [a_ptr4]\n"
- "ldr d22, [a_ptr5]\n"
+ "ldr d5, [a_ptr1]\n"
+ "ldr d8, [a_ptr2]\n"
+ "ldr d11, [a_ptr3]\n"
+ "ldr d14, [a_ptr4]\n"
+ "ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
- "ldr s6, [a_ptr1], #0x4\n"
- "ldr s10, [a_ptr2], #0x4\n"
- "ldr s14, [a_ptr3], #0x4\n"
- "ldr s18, [a_ptr4], #0x4\n"
- "ldr s22, [a_ptr5], #0x4\n"
+ "ldr s5, [a_ptr1], #0x4\n"
+ "ldr s8, [a_ptr2], #0x4\n"
+ "ldr s11, [a_ptr3], #0x4\n"
+ "ldr s14, [a_ptr4], #0x4\n"
+ "ldr s17, [a_ptr5], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
- "ld1 {v6.b}[4], [a_ptr1]\n"
- "ld1 {v10.b}[4], [a_ptr2]\n"
- "ld1 {v14.b}[4], [a_ptr3]\n"
- "ld1 {v18.b}[4], [a_ptr4]\n"
- "ld1 {v22.b}[4], [a_ptr5]\n"
+ "ld1 {v5.b}[4], [a_ptr1]\n"
+ "ld1 {v8.b}[4], [a_ptr2]\n"
+ "ld1 {v11.b}[4], [a_ptr3]\n"
+ "ld1 {v14.b}[4], [a_ptr4]\n"
+ "ld1 {v17.b}[4], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[2], [a_ptr1], #2\n"
- "ld1 {v10.h}[2], [a_ptr2], #2\n"
- "ld1 {v14.h}[2], [a_ptr3], #2\n"
- "ld1 {v18.h}[2], [a_ptr4], #2\n"
- "ld1 {v22.h}[2], [a_ptr5], #2\n"
+ "ld1 {v5.h}[2], [a_ptr1], #2\n"
+ "ld1 {v8.h}[2], [a_ptr2], #2\n"
+ "ld1 {v11.h}[2], [a_ptr3], #2\n"
+ "ld1 {v14.h}[2], [a_ptr4], #2\n"
+ "ld1 {v17.h}[2], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[6], [%[a_ptr0]]\n"
- "ld1 {v6.b}[6], [a_ptr1]\n"
- "ld1 {v10.b}[6], [a_ptr2]\n"
- "ld1 {v14.b}[6], [a_ptr3]\n"
- "ld1 {v18.b}[6], [a_ptr4]\n"
- "ld1 {v22.b}[6], [a_ptr5]\n"
+ "ld1 {v5.b}[6], [a_ptr1]\n"
+ "ld1 {v8.b}[6], [a_ptr2]\n"
+ "ld1 {v11.b}[6], [a_ptr3]\n"
+ "ld1 {v14.b}[6], [a_ptr4]\n"
+ "ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v23.d[1], temploadreg3\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -918,390 +961,408 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr q21, [a_ptr5], #0x10\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v2.s}[2], [%[a_ptr0]]\n"
- "ld1 {v6.s}[2], [a_ptr1]\n"
- "ld1 {v10.s}[2], [a_ptr2]\n"
- "ld1 {v14.s}[2], [a_ptr3]\n"
- "ld1 {v18.s}[2], [a_ptr4]\n"
- "ld1 {v22.s}[2], [a_ptr5]\n"
+ "ld1 {v5.s}[2], [a_ptr1]\n"
+ "ld1 {v8.s}[2], [a_ptr2]\n"
+ "ld1 {v11.s}[2], [a_ptr3]\n"
+ "ld1 {v14.s}[2], [a_ptr4]\n"
+ "ld1 {v17.s}[2], [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[8], [%[a_ptr0]]\n"
- "ld1 {v6.b}[8], [a_ptr1]\n"
- "ld1 {v10.b}[8], [a_ptr2]\n"
- "ld1 {v14.b}[8], [a_ptr3]\n"
- "ld1 {v18.b}[8], [a_ptr4]\n"
- "ld1 {v22.b}[8], [a_ptr5]\n"
+ "ld1 {v5.b}[8], [a_ptr1]\n"
+ "ld1 {v8.b}[8], [a_ptr2]\n"
+ "ld1 {v11.b}[8], [a_ptr3]\n"
+ "ld1 {v14.b}[8], [a_ptr4]\n"
+ "ld1 {v17.b}[8], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[4], [a_ptr1], #2\n"
- "ld1 {v10.h}[4], [a_ptr2], #2\n"
- "ld1 {v14.h}[4], [a_ptr3], #2\n"
- "ld1 {v18.h}[4], [a_ptr4], #2\n"
- "ld1 {v22.h}[4], [a_ptr5], #2\n"
+ "ld1 {v5.h}[4], [a_ptr1], #2\n"
+ "ld1 {v8.h}[4], [a_ptr2], #2\n"
+ "ld1 {v11.h}[4], [a_ptr3], #2\n"
+ "ld1 {v14.h}[4], [a_ptr4], #2\n"
+ "ld1 {v17.h}[4], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[10], [%[a_ptr0]]\n"
- "ld1 {v6.b}[10], [a_ptr1]\n"
- "ld1 {v10.b}[10], [a_ptr2]\n"
- "ld1 {v14.b}[10], [a_ptr3]\n"
- "ld1 {v18.b}[10], [a_ptr4]\n"
- "ld1 {v22.b}[10], [a_ptr5]\n"
+ "ld1 {v5.b}[10], [a_ptr1]\n"
+ "ld1 {v8.b}[10], [a_ptr2]\n"
+ "ld1 {v11.b}[10], [a_ptr3]\n"
+ "ld1 {v14.b}[10], [a_ptr4]\n"
+ "ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
"ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ins v23.d[1], temploadreg3\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1369,419 +1430,437 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
- "ldr q6, [a_ptr1]\n"
- "ldr q10, [a_ptr2]\n"
- "ldr q14, [a_ptr3]\n"
- "ldr q18, [a_ptr4]\n"
- "ldr q22, [a_ptr5]\n"
+ "ldr q5, [a_ptr1]\n"
+ "ldr q8, [a_ptr2]\n"
+ "ldr q11, [a_ptr3]\n"
+ "ldr q14, [a_ptr4]\n"
+ "ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"ld1 {v2.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v6.s}[2], [a_ptr1], #4\n"
- "ld1 {v10.s}[2], [a_ptr2], #4\n"
- "ld1 {v14.s}[2], [a_ptr3], #4\n"
- "ld1 {v18.s}[2], [a_ptr4], #4\n"
- "ld1 {v22.s}[2], [a_ptr5], #4\n"
+ "ld1 {v5.s}[2], [a_ptr1], #4\n"
+ "ld1 {v8.s}[2], [a_ptr2], #4\n"
+ "ld1 {v11.s}[2], [a_ptr3], #4\n"
+ "ld1 {v14.s}[2], [a_ptr4], #4\n"
+ "ld1 {v17.s}[2], [a_ptr5], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
- "ld1 {v6.b}[12], [a_ptr1]\n"
- "ld1 {v10.b}[12], [a_ptr2]\n"
- "ld1 {v14.b}[12], [a_ptr3]\n"
- "ld1 {v18.b}[12], [a_ptr4]\n"
- "ld1 {v22.b}[12], [a_ptr5]\n"
+ "ld1 {v5.b}[12], [a_ptr1]\n"
+ "ld1 {v8.b}[12], [a_ptr2]\n"
+ "ld1 {v11.b}[12], [a_ptr3]\n"
+ "ld1 {v14.b}[12], [a_ptr4]\n"
+ "ld1 {v17.b}[12], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[6], [a_ptr1], #2\n"
- "ld1 {v10.h}[6], [a_ptr2], #2\n"
- "ld1 {v14.h}[6], [a_ptr3], #2\n"
- "ld1 {v18.h}[6], [a_ptr4], #2\n"
- "ld1 {v22.h}[6], [a_ptr5], #2\n"
+ "ld1 {v5.h}[6], [a_ptr1], #2\n"
+ "ld1 {v8.h}[6], [a_ptr2], #2\n"
+ "ld1 {v11.h}[6], [a_ptr3], #2\n"
+ "ld1 {v14.h}[6], [a_ptr4], #2\n"
+ "ld1 {v17.h}[6], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[14], [%[a_ptr0]]\n"
- "ld1 {v6.b}[14], [a_ptr1]\n"
- "ld1 {v10.b}[14], [a_ptr2]\n"
- "ld1 {v14.b}[14], [a_ptr3]\n"
- "ld1 {v18.b}[14], [a_ptr4]\n"
- "ld1 {v22.b}[14], [a_ptr5]\n"
+ "ld1 {v5.b}[14], [a_ptr1]\n"
+ "ld1 {v8.b}[14], [a_ptr2]\n"
+ "ld1 {v11.b}[14], [a_ptr3]\n"
+ "ld1 {v14.b}[14], [a_ptr4]\n"
+ "ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ins v25.d[1], temploadreg1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
"ins v25.d[1], temploadreg1\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1914,104 +1993,104 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2023,265 +2102,283 @@
"ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr d24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2420,111 +2517,111 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2535,282 +2632,300 @@
"ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ins v25.d[1], temploadreg1\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr d24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr d24, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
"ins v25.d[1], temploadreg1\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2949,119 +3064,119 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -3073,299 +3188,317 @@
"ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr d24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3511,126 +3644,126 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -3641,316 +3774,334 @@
"ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ins v25.d[1], temploadreg1\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr d24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ "ldr d24, [%[b_ptr0]]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
"ins v25.d[1], temploadreg1\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp
index 64ab6f0..d4082130 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp
@@ -25,15 +25,16 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- UNUSED(beta);
- const long loops_count = (N / 4) - 1;
+void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+ const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
const long odds_count = K % 4;
@@ -93,313 +94,322 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr s2, [%[a_ptr0]]\n"
- "ldr s6, [a_ptr1]\n"
- "ldr s10, [a_ptr2]\n"
- "ldr s14, [a_ptr3]\n"
- "ldr s18, [a_ptr4]\n"
- "ldr s22, [a_ptr5]\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
- "ldr b6, [a_ptr1]\n"
- "ldr b10, [a_ptr2]\n"
- "ldr b14, [a_ptr3]\n"
- "ldr b18, [a_ptr4]\n"
- "ldr b22, [a_ptr5]\n"
+ "ldr b5, [a_ptr1]\n"
+ "ldr b8, [a_ptr2]\n"
+ "ldr b11, [a_ptr3]\n"
+ "ldr b14, [a_ptr4]\n"
+ "ldr b17, [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ldr h2, [%[a_ptr0]], #0x2\n"
- "ldr h6, [a_ptr1], #0x2\n"
- "ldr h10, [a_ptr2], #0x2\n"
- "ldr h14, [a_ptr3], #0x2\n"
- "ldr h18, [a_ptr4], #0x2\n"
- "ldr h22, [a_ptr5], #0x2\n"
+ "ldr h5, [a_ptr1], #0x2\n"
+ "ldr h8, [a_ptr2], #0x2\n"
+ "ldr h11, [a_ptr3], #0x2\n"
+ "ldr h14, [a_ptr4], #0x2\n"
+ "ldr h17, [a_ptr5], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[2], [%[a_ptr0]]\n"
- "ld1 {v6.b}[2], [a_ptr1]\n"
- "ld1 {v10.b}[2], [a_ptr2]\n"
- "ld1 {v14.b}[2], [a_ptr3]\n"
- "ld1 {v18.b}[2], [a_ptr4]\n"
- "ld1 {v22.b}[2], [a_ptr5]\n"
+ "ld1 {v5.b}[2], [a_ptr1]\n"
+ "ld1 {v8.b}[2], [a_ptr2]\n"
+ "ld1 {v11.b}[2], [a_ptr3]\n"
+ "ld1 {v14.b}[2], [a_ptr4]\n"
+ "ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -459,340 +469,349 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
- "ldr d6, [a_ptr1]\n"
- "ldr d10, [a_ptr2]\n"
- "ldr d14, [a_ptr3]\n"
- "ldr d18, [a_ptr4]\n"
- "ldr d22, [a_ptr5]\n"
+ "ldr d5, [a_ptr1]\n"
+ "ldr d8, [a_ptr2]\n"
+ "ldr d11, [a_ptr3]\n"
+ "ldr d14, [a_ptr4]\n"
+ "ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
- "ldr s6, [a_ptr1], #0x4\n"
- "ldr s10, [a_ptr2], #0x4\n"
- "ldr s14, [a_ptr3], #0x4\n"
- "ldr s18, [a_ptr4], #0x4\n"
- "ldr s22, [a_ptr5], #0x4\n"
+ "ldr s5, [a_ptr1], #0x4\n"
+ "ldr s8, [a_ptr2], #0x4\n"
+ "ldr s11, [a_ptr3], #0x4\n"
+ "ldr s14, [a_ptr4], #0x4\n"
+ "ldr s17, [a_ptr5], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
- "ld1 {v6.b}[4], [a_ptr1]\n"
- "ld1 {v10.b}[4], [a_ptr2]\n"
- "ld1 {v14.b}[4], [a_ptr3]\n"
- "ld1 {v18.b}[4], [a_ptr4]\n"
- "ld1 {v22.b}[4], [a_ptr5]\n"
+ "ld1 {v5.b}[4], [a_ptr1]\n"
+ "ld1 {v8.b}[4], [a_ptr2]\n"
+ "ld1 {v11.b}[4], [a_ptr3]\n"
+ "ld1 {v14.b}[4], [a_ptr4]\n"
+ "ld1 {v17.b}[4], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[2], [a_ptr1], #2\n"
- "ld1 {v10.h}[2], [a_ptr2], #2\n"
- "ld1 {v14.h}[2], [a_ptr3], #2\n"
- "ld1 {v18.h}[2], [a_ptr4], #2\n"
- "ld1 {v22.h}[2], [a_ptr5], #2\n"
+ "ld1 {v5.h}[2], [a_ptr1], #2\n"
+ "ld1 {v8.h}[2], [a_ptr2], #2\n"
+ "ld1 {v11.h}[2], [a_ptr3], #2\n"
+ "ld1 {v14.h}[2], [a_ptr4], #2\n"
+ "ld1 {v17.h}[2], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[6], [%[a_ptr0]]\n"
- "ld1 {v6.b}[6], [a_ptr1]\n"
- "ld1 {v10.b}[6], [a_ptr2]\n"
- "ld1 {v14.b}[6], [a_ptr3]\n"
- "ld1 {v18.b}[6], [a_ptr4]\n"
- "ld1 {v22.b}[6], [a_ptr5]\n"
+ "ld1 {v5.b}[6], [a_ptr1]\n"
+ "ld1 {v8.b}[6], [a_ptr2]\n"
+ "ld1 {v11.b}[6], [a_ptr3]\n"
+ "ld1 {v14.b}[6], [a_ptr4]\n"
+ "ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
- "subs %[loops], %[loops], #0x1\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -852,364 +871,370 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr q21, [a_ptr5], #0x10\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v2.s}[2], [%[a_ptr0]]\n"
- "ld1 {v6.s}[2], [a_ptr1]\n"
- "ld1 {v10.s}[2], [a_ptr2]\n"
- "ld1 {v14.s}[2], [a_ptr3]\n"
- "ld1 {v18.s}[2], [a_ptr4]\n"
- "ld1 {v22.s}[2], [a_ptr5]\n"
+ "ld1 {v5.s}[2], [a_ptr1]\n"
+ "ld1 {v8.s}[2], [a_ptr2]\n"
+ "ld1 {v11.s}[2], [a_ptr3]\n"
+ "ld1 {v14.s}[2], [a_ptr4]\n"
+ "ld1 {v17.s}[2], [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[8], [%[a_ptr0]]\n"
- "ld1 {v6.b}[8], [a_ptr1]\n"
- "ld1 {v10.b}[8], [a_ptr2]\n"
- "ld1 {v14.b}[8], [a_ptr3]\n"
- "ld1 {v18.b}[8], [a_ptr4]\n"
- "ld1 {v22.b}[8], [a_ptr5]\n"
+ "ld1 {v5.b}[8], [a_ptr1]\n"
+ "ld1 {v8.b}[8], [a_ptr2]\n"
+ "ld1 {v11.b}[8], [a_ptr3]\n"
+ "ld1 {v14.b}[8], [a_ptr4]\n"
+ "ld1 {v17.b}[8], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[4], [a_ptr1], #2\n"
- "ld1 {v10.h}[4], [a_ptr2], #2\n"
- "ld1 {v14.h}[4], [a_ptr3], #2\n"
- "ld1 {v18.h}[4], [a_ptr4], #2\n"
- "ld1 {v22.h}[4], [a_ptr5], #2\n"
+ "ld1 {v5.h}[4], [a_ptr1], #2\n"
+ "ld1 {v8.h}[4], [a_ptr2], #2\n"
+ "ld1 {v11.h}[4], [a_ptr3], #2\n"
+ "ld1 {v14.h}[4], [a_ptr4], #2\n"
+ "ld1 {v17.h}[4], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[10], [%[a_ptr0]]\n"
- "ld1 {v6.b}[10], [a_ptr1]\n"
- "ld1 {v10.b}[10], [a_ptr2]\n"
- "ld1 {v14.b}[10], [a_ptr3]\n"
- "ld1 {v18.b}[10], [a_ptr4]\n"
- "ld1 {v22.b}[10], [a_ptr5]\n"
+ "ld1 {v5.b}[10], [a_ptr1]\n"
+ "ld1 {v8.b}[10], [a_ptr2]\n"
+ "ld1 {v11.b}[10], [a_ptr3]\n"
+ "ld1 {v14.b}[10], [a_ptr4]\n"
+ "ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1269,391 +1294,397 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
- "ldr q6, [a_ptr1]\n"
- "ldr q10, [a_ptr2]\n"
- "ldr q14, [a_ptr3]\n"
- "ldr q18, [a_ptr4]\n"
- "ldr q22, [a_ptr5]\n"
+ "ldr q5, [a_ptr1]\n"
+ "ldr q8, [a_ptr2]\n"
+ "ldr q11, [a_ptr3]\n"
+ "ldr q14, [a_ptr4]\n"
+ "ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"ld1 {v2.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v6.s}[2], [a_ptr1], #4\n"
- "ld1 {v10.s}[2], [a_ptr2], #4\n"
- "ld1 {v14.s}[2], [a_ptr3], #4\n"
- "ld1 {v18.s}[2], [a_ptr4], #4\n"
- "ld1 {v22.s}[2], [a_ptr5], #4\n"
+ "ld1 {v5.s}[2], [a_ptr1], #4\n"
+ "ld1 {v8.s}[2], [a_ptr2], #4\n"
+ "ld1 {v11.s}[2], [a_ptr3], #4\n"
+ "ld1 {v14.s}[2], [a_ptr4], #4\n"
+ "ld1 {v17.s}[2], [a_ptr5], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
- "ld1 {v6.b}[12], [a_ptr1]\n"
- "ld1 {v10.b}[12], [a_ptr2]\n"
- "ld1 {v14.b}[12], [a_ptr3]\n"
- "ld1 {v18.b}[12], [a_ptr4]\n"
- "ld1 {v22.b}[12], [a_ptr5]\n"
+ "ld1 {v5.b}[12], [a_ptr1]\n"
+ "ld1 {v8.b}[12], [a_ptr2]\n"
+ "ld1 {v11.b}[12], [a_ptr3]\n"
+ "ld1 {v14.b}[12], [a_ptr4]\n"
+ "ld1 {v17.b}[12], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[6], [a_ptr1], #2\n"
- "ld1 {v10.h}[6], [a_ptr2], #2\n"
- "ld1 {v14.h}[6], [a_ptr3], #2\n"
- "ld1 {v18.h}[6], [a_ptr4], #2\n"
- "ld1 {v22.h}[6], [a_ptr5], #2\n"
+ "ld1 {v5.h}[6], [a_ptr1], #2\n"
+ "ld1 {v8.h}[6], [a_ptr2], #2\n"
+ "ld1 {v11.h}[6], [a_ptr3], #2\n"
+ "ld1 {v14.h}[6], [a_ptr4], #2\n"
+ "ld1 {v17.h}[6], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[14], [%[a_ptr0]]\n"
- "ld1 {v6.b}[14], [a_ptr1]\n"
- "ld1 {v10.b}[14], [a_ptr2]\n"
- "ld1 {v14.b}[14], [a_ptr3]\n"
- "ld1 {v18.b}[14], [a_ptr4]\n"
- "ld1 {v22.b}[14], [a_ptr5]\n"
+ "ld1 {v5.b}[14], [a_ptr1]\n"
+ "ld1 {v8.b}[14], [a_ptr2]\n"
+ "ld1 {v11.b}[14], [a_ptr3]\n"
+ "ld1 {v14.b}[14], [a_ptr4]\n"
+ "ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
- "subs %[loops], %[loops], #0x1\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1778,344 +1809,362 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2246,365 +2295,383 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2735,389 +2802,407 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"movi v26.4s, #0\n"
- "str q27, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3255,410 +3340,428 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
"b.ne 8b\n"
"7:\n"
- "str q26, [%[c_ptr0]], #0x10\n"
+ "str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1], #0x10\n"
+ "str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
- "str q28, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "str q29, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
- ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
- "str q30, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "str q31, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp
index 6615f1f..02be0bd 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 ARM Limited.
+ * Copyright (c) 2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -31,8 +31,8 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
-void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
class smallK_hybrid_s8s32_dot_4x8
{
@@ -40,10 +40,10 @@
typedef int8_t operand_type;
typedef int32_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+ typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 8;
}
@@ -53,11 +53,26 @@
return 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {};
// Default to the generic kernel
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp
index 4ed2bc8..c742fcc 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp
@@ -25,15 +25,16 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- UNUSED(beta);
- const long loops_count = (N / 4) - 1;
+void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+ const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
const long odds_count = K % 4;
@@ -114,47 +115,47 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr s0, [%[a_ptr0]]\n"
- "ldr s2, [a_ptr1]\n"
- "ldr s4, [a_ptr2]\n"
- "ldr s6, [a_ptr3]\n"
- "ldr s8, [a_ptr4]\n"
- "ldr s10, [a_ptr5]\n"
- "ldr s12, [a_ptr6]\n"
- "ldr s14, [a_ptr7]\n"
+ "ldr s1, [a_ptr1]\n"
+ "ldr s2, [a_ptr2]\n"
+ "ldr s3, [a_ptr3]\n"
+ "ldr s4, [a_ptr4]\n"
+ "ldr s5, [a_ptr5]\n"
+ "ldr s6, [a_ptr6]\n"
+ "ldr s7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b0, [%[a_ptr0]]\n"
- "ldr b2, [a_ptr1]\n"
- "ldr b4, [a_ptr2]\n"
- "ldr b6, [a_ptr3]\n"
- "ldr b8, [a_ptr4]\n"
- "ldr b10, [a_ptr5]\n"
- "ldr b12, [a_ptr6]\n"
- "ldr b14, [a_ptr7]\n"
+ "ldr b1, [a_ptr1]\n"
+ "ldr b2, [a_ptr2]\n"
+ "ldr b3, [a_ptr3]\n"
+ "ldr b4, [a_ptr4]\n"
+ "ldr b5, [a_ptr5]\n"
+ "ldr b6, [a_ptr6]\n"
+ "ldr b7, [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ldr h0, [%[a_ptr0]], #0x2\n"
- "ldr h2, [a_ptr1], #0x2\n"
- "ldr h4, [a_ptr2], #0x2\n"
- "ldr h6, [a_ptr3], #0x2\n"
- "ldr h8, [a_ptr4], #0x2\n"
- "ldr h10, [a_ptr5], #0x2\n"
- "ldr h12, [a_ptr6], #0x2\n"
- "ldr h14, [a_ptr7], #0x2\n"
+ "ldr h1, [a_ptr1], #0x2\n"
+ "ldr h2, [a_ptr2], #0x2\n"
+ "ldr h3, [a_ptr3], #0x2\n"
+ "ldr h4, [a_ptr4], #0x2\n"
+ "ldr h5, [a_ptr5], #0x2\n"
+ "ldr h6, [a_ptr6], #0x2\n"
+ "ldr h7, [a_ptr7], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v2.b}[2], [a_ptr1]\n"
- "ld1 {v4.b}[2], [a_ptr2]\n"
- "ld1 {v6.b}[2], [a_ptr3]\n"
- "ld1 {v8.b}[2], [a_ptr4]\n"
- "ld1 {v10.b}[2], [a_ptr5]\n"
- "ld1 {v12.b}[2], [a_ptr6]\n"
- "ld1 {v14.b}[2], [a_ptr7]\n"
+ "ld1 {v1.b}[2], [a_ptr1]\n"
+ "ld1 {v2.b}[2], [a_ptr2]\n"
+ "ld1 {v3.b}[2], [a_ptr3]\n"
+ "ld1 {v4.b}[2], [a_ptr4]\n"
+ "ld1 {v5.b}[2], [a_ptr5]\n"
+ "ld1 {v6.b}[2], [a_ptr6]\n"
+ "ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -172,94 +173,118 @@
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "ins v16.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "movi v24.4s, #0\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "movi v24.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v16.d[1], temploadreg0\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -348,55 +373,55 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr d0, [%[a_ptr0]]\n"
- "ldr d2, [a_ptr1]\n"
- "ldr d4, [a_ptr2]\n"
- "ldr d6, [a_ptr3]\n"
- "ldr d8, [a_ptr4]\n"
- "ldr d10, [a_ptr5]\n"
- "ldr d12, [a_ptr6]\n"
- "ldr d14, [a_ptr7]\n"
+ "ldr d1, [a_ptr1]\n"
+ "ldr d2, [a_ptr2]\n"
+ "ldr d3, [a_ptr3]\n"
+ "ldr d4, [a_ptr4]\n"
+ "ldr d5, [a_ptr5]\n"
+ "ldr d6, [a_ptr6]\n"
+ "ldr d7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr s0, [%[a_ptr0]], #0x4\n"
- "ldr s2, [a_ptr1], #0x4\n"
- "ldr s4, [a_ptr2], #0x4\n"
- "ldr s6, [a_ptr3], #0x4\n"
- "ldr s8, [a_ptr4], #0x4\n"
- "ldr s10, [a_ptr5], #0x4\n"
- "ldr s12, [a_ptr6], #0x4\n"
- "ldr s14, [a_ptr7], #0x4\n"
+ "ldr s1, [a_ptr1], #0x4\n"
+ "ldr s2, [a_ptr2], #0x4\n"
+ "ldr s3, [a_ptr3], #0x4\n"
+ "ldr s4, [a_ptr4], #0x4\n"
+ "ldr s5, [a_ptr5], #0x4\n"
+ "ldr s6, [a_ptr6], #0x4\n"
+ "ldr s7, [a_ptr7], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[4], [%[a_ptr0]]\n"
- "ld1 {v2.b}[4], [a_ptr1]\n"
- "ld1 {v4.b}[4], [a_ptr2]\n"
- "ld1 {v6.b}[4], [a_ptr3]\n"
- "ld1 {v8.b}[4], [a_ptr4]\n"
- "ld1 {v10.b}[4], [a_ptr5]\n"
- "ld1 {v12.b}[4], [a_ptr6]\n"
- "ld1 {v14.b}[4], [a_ptr7]\n"
+ "ld1 {v1.b}[4], [a_ptr1]\n"
+ "ld1 {v2.b}[4], [a_ptr2]\n"
+ "ld1 {v3.b}[4], [a_ptr3]\n"
+ "ld1 {v4.b}[4], [a_ptr4]\n"
+ "ld1 {v5.b}[4], [a_ptr5]\n"
+ "ld1 {v6.b}[4], [a_ptr6]\n"
+ "ld1 {v7.b}[4], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[2], [a_ptr1], #2\n"
- "ld1 {v4.h}[2], [a_ptr2], #2\n"
- "ld1 {v6.h}[2], [a_ptr3], #2\n"
- "ld1 {v8.h}[2], [a_ptr4], #2\n"
- "ld1 {v10.h}[2], [a_ptr5], #2\n"
- "ld1 {v12.h}[2], [a_ptr6], #2\n"
- "ld1 {v14.h}[2], [a_ptr7], #2\n"
+ "ld1 {v1.h}[2], [a_ptr1], #2\n"
+ "ld1 {v2.h}[2], [a_ptr2], #2\n"
+ "ld1 {v3.h}[2], [a_ptr3], #2\n"
+ "ld1 {v4.h}[2], [a_ptr4], #2\n"
+ "ld1 {v5.h}[2], [a_ptr5], #2\n"
+ "ld1 {v6.h}[2], [a_ptr6], #2\n"
+ "ld1 {v7.h}[2], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[6], [%[a_ptr0]]\n"
- "ld1 {v2.b}[6], [a_ptr1]\n"
- "ld1 {v4.b}[6], [a_ptr2]\n"
- "ld1 {v6.b}[6], [a_ptr3]\n"
- "ld1 {v8.b}[6], [a_ptr4]\n"
- "ld1 {v10.b}[6], [a_ptr5]\n"
- "ld1 {v12.b}[6], [a_ptr6]\n"
- "ld1 {v14.b}[6], [a_ptr7]\n"
+ "ld1 {v1.b}[6], [a_ptr1]\n"
+ "ld1 {v2.b}[6], [a_ptr2]\n"
+ "ld1 {v3.b}[6], [a_ptr3]\n"
+ "ld1 {v4.b}[6], [a_ptr4]\n"
+ "ld1 {v5.b}[6], [a_ptr5]\n"
+ "ld1 {v6.b}[6], [a_ptr6]\n"
+ "ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -414,23 +439,23 @@
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -441,98 +466,122 @@
"ins v16.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ins v17.d[1], temploadreg1\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q25, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ins v17.d[1], temploadreg1\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -620,56 +669,56 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v0.s}[2], [%[a_ptr0]]\n"
- "ld1 {v2.s}[2], [a_ptr1]\n"
- "ld1 {v4.s}[2], [a_ptr2]\n"
- "ld1 {v6.s}[2], [a_ptr3]\n"
- "ld1 {v8.s}[2], [a_ptr4]\n"
- "ld1 {v10.s}[2], [a_ptr5]\n"
- "ld1 {v12.s}[2], [a_ptr6]\n"
- "ld1 {v14.s}[2], [a_ptr7]\n"
+ "ld1 {v1.s}[2], [a_ptr1]\n"
+ "ld1 {v2.s}[2], [a_ptr2]\n"
+ "ld1 {v3.s}[2], [a_ptr3]\n"
+ "ld1 {v4.s}[2], [a_ptr4]\n"
+ "ld1 {v5.s}[2], [a_ptr5]\n"
+ "ld1 {v6.s}[2], [a_ptr6]\n"
+ "ld1 {v7.s}[2], [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[8], [%[a_ptr0]]\n"
- "ld1 {v2.b}[8], [a_ptr1]\n"
- "ld1 {v4.b}[8], [a_ptr2]\n"
- "ld1 {v6.b}[8], [a_ptr3]\n"
- "ld1 {v8.b}[8], [a_ptr4]\n"
- "ld1 {v10.b}[8], [a_ptr5]\n"
- "ld1 {v12.b}[8], [a_ptr6]\n"
- "ld1 {v14.b}[8], [a_ptr7]\n"
+ "ld1 {v1.b}[8], [a_ptr1]\n"
+ "ld1 {v2.b}[8], [a_ptr2]\n"
+ "ld1 {v3.b}[8], [a_ptr3]\n"
+ "ld1 {v4.b}[8], [a_ptr4]\n"
+ "ld1 {v5.b}[8], [a_ptr5]\n"
+ "ld1 {v6.b}[8], [a_ptr6]\n"
+ "ld1 {v7.b}[8], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[4], [a_ptr1], #2\n"
- "ld1 {v4.h}[4], [a_ptr2], #2\n"
- "ld1 {v6.h}[4], [a_ptr3], #2\n"
- "ld1 {v8.h}[4], [a_ptr4], #2\n"
- "ld1 {v10.h}[4], [a_ptr5], #2\n"
- "ld1 {v12.h}[4], [a_ptr6], #2\n"
- "ld1 {v14.h}[4], [a_ptr7], #2\n"
+ "ld1 {v1.h}[4], [a_ptr1], #2\n"
+ "ld1 {v2.h}[4], [a_ptr2], #2\n"
+ "ld1 {v3.h}[4], [a_ptr3], #2\n"
+ "ld1 {v4.h}[4], [a_ptr4], #2\n"
+ "ld1 {v5.h}[4], [a_ptr5], #2\n"
+ "ld1 {v6.h}[4], [a_ptr6], #2\n"
+ "ld1 {v7.h}[4], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[10], [%[a_ptr0]]\n"
- "ld1 {v2.b}[10], [a_ptr1]\n"
- "ld1 {v4.b}[10], [a_ptr2]\n"
- "ld1 {v6.b}[10], [a_ptr3]\n"
- "ld1 {v8.b}[10], [a_ptr4]\n"
- "ld1 {v10.b}[10], [a_ptr5]\n"
- "ld1 {v12.b}[10], [a_ptr6]\n"
- "ld1 {v14.b}[10], [a_ptr7]\n"
+ "ld1 {v1.b}[10], [a_ptr1]\n"
+ "ld1 {v2.b}[10], [a_ptr2]\n"
+ "ld1 {v3.b}[10], [a_ptr3]\n"
+ "ld1 {v4.b}[10], [a_ptr4]\n"
+ "ld1 {v5.b}[10], [a_ptr5]\n"
+ "ld1 {v6.b}[10], [a_ptr6]\n"
+ "ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -687,32 +736,32 @@
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -726,117 +775,141 @@
"ins v17.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ins v18.d[1], temploadreg2\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q25, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "str q26, [c_ptr2], #0x10\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ins v18.d[1], temploadreg2\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -925,63 +998,63 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]]\n"
- "ldr q2, [a_ptr1]\n"
- "ldr q4, [a_ptr2]\n"
- "ldr q6, [a_ptr3]\n"
- "ldr q8, [a_ptr4]\n"
- "ldr q10, [a_ptr5]\n"
- "ldr q12, [a_ptr6]\n"
- "ldr q14, [a_ptr7]\n"
+ "ldr q1, [a_ptr1]\n"
+ "ldr q2, [a_ptr2]\n"
+ "ldr q3, [a_ptr3]\n"
+ "ldr q4, [a_ptr4]\n"
+ "ldr q5, [a_ptr5]\n"
+ "ldr q6, [a_ptr6]\n"
+ "ldr q7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"ld1 {v0.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v2.s}[2], [a_ptr1], #4\n"
- "ld1 {v4.s}[2], [a_ptr2], #4\n"
- "ld1 {v6.s}[2], [a_ptr3], #4\n"
- "ld1 {v8.s}[2], [a_ptr4], #4\n"
- "ld1 {v10.s}[2], [a_ptr5], #4\n"
- "ld1 {v12.s}[2], [a_ptr6], #4\n"
- "ld1 {v14.s}[2], [a_ptr7], #4\n"
+ "ld1 {v1.s}[2], [a_ptr1], #4\n"
+ "ld1 {v2.s}[2], [a_ptr2], #4\n"
+ "ld1 {v3.s}[2], [a_ptr3], #4\n"
+ "ld1 {v4.s}[2], [a_ptr4], #4\n"
+ "ld1 {v5.s}[2], [a_ptr5], #4\n"
+ "ld1 {v6.s}[2], [a_ptr6], #4\n"
+ "ld1 {v7.s}[2], [a_ptr7], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[12], [%[a_ptr0]]\n"
- "ld1 {v2.b}[12], [a_ptr1]\n"
- "ld1 {v4.b}[12], [a_ptr2]\n"
- "ld1 {v6.b}[12], [a_ptr3]\n"
- "ld1 {v8.b}[12], [a_ptr4]\n"
- "ld1 {v10.b}[12], [a_ptr5]\n"
- "ld1 {v12.b}[12], [a_ptr6]\n"
- "ld1 {v14.b}[12], [a_ptr7]\n"
+ "ld1 {v1.b}[12], [a_ptr1]\n"
+ "ld1 {v2.b}[12], [a_ptr2]\n"
+ "ld1 {v3.b}[12], [a_ptr3]\n"
+ "ld1 {v4.b}[12], [a_ptr4]\n"
+ "ld1 {v5.b}[12], [a_ptr5]\n"
+ "ld1 {v6.b}[12], [a_ptr6]\n"
+ "ld1 {v7.b}[12], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[6], [a_ptr1], #2\n"
- "ld1 {v4.h}[6], [a_ptr2], #2\n"
- "ld1 {v6.h}[6], [a_ptr3], #2\n"
- "ld1 {v8.h}[6], [a_ptr4], #2\n"
- "ld1 {v10.h}[6], [a_ptr5], #2\n"
- "ld1 {v12.h}[6], [a_ptr6], #2\n"
- "ld1 {v14.h}[6], [a_ptr7], #2\n"
+ "ld1 {v1.h}[6], [a_ptr1], #2\n"
+ "ld1 {v2.h}[6], [a_ptr2], #2\n"
+ "ld1 {v3.h}[6], [a_ptr3], #2\n"
+ "ld1 {v4.h}[6], [a_ptr4], #2\n"
+ "ld1 {v5.h}[6], [a_ptr5], #2\n"
+ "ld1 {v6.h}[6], [a_ptr6], #2\n"
+ "ld1 {v7.h}[6], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[14], [%[a_ptr0]]\n"
- "ld1 {v2.b}[14], [a_ptr1]\n"
- "ld1 {v4.b}[14], [a_ptr2]\n"
- "ld1 {v6.b}[14], [a_ptr3]\n"
- "ld1 {v8.b}[14], [a_ptr4]\n"
- "ld1 {v10.b}[14], [a_ptr5]\n"
- "ld1 {v12.b}[14], [a_ptr6]\n"
- "ld1 {v14.b}[14], [a_ptr7]\n"
+ "ld1 {v1.b}[14], [a_ptr1]\n"
+ "ld1 {v2.b}[14], [a_ptr2]\n"
+ "ld1 {v3.b}[14], [a_ptr3]\n"
+ "ld1 {v4.b}[14], [a_ptr4]\n"
+ "ld1 {v5.b}[14], [a_ptr5]\n"
+ "ld1 {v6.b}[14], [a_ptr6]\n"
+ "ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -999,41 +1072,41 @@
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1050,136 +1123,160 @@
"ins v18.d[1], temploadreg2\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ins v19.d[1], temploadreg3\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q25, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
+ "movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
- "movi v26.4s, #0\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ins v19.d[1], temploadreg3\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1334,50 +1431,50 @@
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1397,155 +1494,179 @@
"ins v19.d[1], temploadreg3\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ins v20.d[1], temploadreg0\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q25, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
+ "movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
- "movi v26.4s, #0\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ins v20.d[1], temploadreg0\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1708,59 +1829,59 @@
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1783,174 +1904,198 @@
"ins v20.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ins v21.d[1], temploadreg1\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q25, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
+ "movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
- "movi v26.4s, #0\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
"ins v20.d[1], temploadreg0\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"ldr d21, [%[b_ptr0], #0x50]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ins v21.d[1], temploadreg1\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2113,68 +2258,68 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2200,193 +2345,217 @@
"ins v21.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ins v22.d[1], temploadreg2\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q25, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
+ "movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
- "movi v26.4s, #0\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
"ins v20.d[1], temploadreg0\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"ldr d21, [%[b_ptr0], #0x50]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
"ins v21.d[1], temploadreg1\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"ldr d22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ins v22.d[1], temploadreg2\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2558,77 +2727,77 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"ldr q23, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2657,212 +2826,236 @@
"ins v22.d[1], temploadreg2\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ins v23.d[1], temploadreg3\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "str q25, [c_ptr1], #0x10\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
+ "movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
- "movi v26.4s, #0\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
"ins v20.d[1], temploadreg0\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"ldr d21, [%[b_ptr0], #0x50]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
"ins v21.d[1], temploadreg1\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"ldr d22, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
"ins v22.d[1], temploadreg2\n"
- ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
"ldr d23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ins v23.d[1], temploadreg3\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp
index 1eb58e1..2df2c30 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp
@@ -25,15 +25,16 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- UNUSED(beta);
- const long loops_count = (N / 4) - 1;
+void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+ const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
const long odds_count = K % 4;
@@ -110,47 +111,47 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr s0, [%[a_ptr0]]\n"
- "ldr s2, [a_ptr1]\n"
- "ldr s4, [a_ptr2]\n"
- "ldr s6, [a_ptr3]\n"
- "ldr s8, [a_ptr4]\n"
- "ldr s10, [a_ptr5]\n"
- "ldr s12, [a_ptr6]\n"
- "ldr s14, [a_ptr7]\n"
+ "ldr s1, [a_ptr1]\n"
+ "ldr s2, [a_ptr2]\n"
+ "ldr s3, [a_ptr3]\n"
+ "ldr s4, [a_ptr4]\n"
+ "ldr s5, [a_ptr5]\n"
+ "ldr s6, [a_ptr6]\n"
+ "ldr s7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b0, [%[a_ptr0]]\n"
- "ldr b2, [a_ptr1]\n"
- "ldr b4, [a_ptr2]\n"
- "ldr b6, [a_ptr3]\n"
- "ldr b8, [a_ptr4]\n"
- "ldr b10, [a_ptr5]\n"
- "ldr b12, [a_ptr6]\n"
- "ldr b14, [a_ptr7]\n"
+ "ldr b1, [a_ptr1]\n"
+ "ldr b2, [a_ptr2]\n"
+ "ldr b3, [a_ptr3]\n"
+ "ldr b4, [a_ptr4]\n"
+ "ldr b5, [a_ptr5]\n"
+ "ldr b6, [a_ptr6]\n"
+ "ldr b7, [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ldr h0, [%[a_ptr0]], #0x2\n"
- "ldr h2, [a_ptr1], #0x2\n"
- "ldr h4, [a_ptr2], #0x2\n"
- "ldr h6, [a_ptr3], #0x2\n"
- "ldr h8, [a_ptr4], #0x2\n"
- "ldr h10, [a_ptr5], #0x2\n"
- "ldr h12, [a_ptr6], #0x2\n"
- "ldr h14, [a_ptr7], #0x2\n"
+ "ldr h1, [a_ptr1], #0x2\n"
+ "ldr h2, [a_ptr2], #0x2\n"
+ "ldr h3, [a_ptr3], #0x2\n"
+ "ldr h4, [a_ptr4], #0x2\n"
+ "ldr h5, [a_ptr5], #0x2\n"
+ "ldr h6, [a_ptr6], #0x2\n"
+ "ldr h7, [a_ptr7], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v2.b}[2], [a_ptr1]\n"
- "ld1 {v4.b}[2], [a_ptr2]\n"
- "ld1 {v6.b}[2], [a_ptr3]\n"
- "ld1 {v8.b}[2], [a_ptr4]\n"
- "ld1 {v10.b}[2], [a_ptr5]\n"
- "ld1 {v12.b}[2], [a_ptr6]\n"
- "ld1 {v14.b}[2], [a_ptr7]\n"
+ "ld1 {v1.b}[2], [a_ptr1]\n"
+ "ld1 {v2.b}[2], [a_ptr2]\n"
+ "ld1 {v3.b}[2], [a_ptr3]\n"
+ "ld1 {v4.b}[2], [a_ptr4]\n"
+ "ld1 {v5.b}[2], [a_ptr5]\n"
+ "ld1 {v6.b}[2], [a_ptr6]\n"
+ "ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -168,90 +169,114 @@
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "movi v24.4s, #0\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "movi v24.4s, #0\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -332,55 +357,55 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr d0, [%[a_ptr0]]\n"
- "ldr d2, [a_ptr1]\n"
- "ldr d4, [a_ptr2]\n"
- "ldr d6, [a_ptr3]\n"
- "ldr d8, [a_ptr4]\n"
- "ldr d10, [a_ptr5]\n"
- "ldr d12, [a_ptr6]\n"
- "ldr d14, [a_ptr7]\n"
+ "ldr d1, [a_ptr1]\n"
+ "ldr d2, [a_ptr2]\n"
+ "ldr d3, [a_ptr3]\n"
+ "ldr d4, [a_ptr4]\n"
+ "ldr d5, [a_ptr5]\n"
+ "ldr d6, [a_ptr6]\n"
+ "ldr d7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr s0, [%[a_ptr0]], #0x4\n"
- "ldr s2, [a_ptr1], #0x4\n"
- "ldr s4, [a_ptr2], #0x4\n"
- "ldr s6, [a_ptr3], #0x4\n"
- "ldr s8, [a_ptr4], #0x4\n"
- "ldr s10, [a_ptr5], #0x4\n"
- "ldr s12, [a_ptr6], #0x4\n"
- "ldr s14, [a_ptr7], #0x4\n"
+ "ldr s1, [a_ptr1], #0x4\n"
+ "ldr s2, [a_ptr2], #0x4\n"
+ "ldr s3, [a_ptr3], #0x4\n"
+ "ldr s4, [a_ptr4], #0x4\n"
+ "ldr s5, [a_ptr5], #0x4\n"
+ "ldr s6, [a_ptr6], #0x4\n"
+ "ldr s7, [a_ptr7], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[4], [%[a_ptr0]]\n"
- "ld1 {v2.b}[4], [a_ptr1]\n"
- "ld1 {v4.b}[4], [a_ptr2]\n"
- "ld1 {v6.b}[4], [a_ptr3]\n"
- "ld1 {v8.b}[4], [a_ptr4]\n"
- "ld1 {v10.b}[4], [a_ptr5]\n"
- "ld1 {v12.b}[4], [a_ptr6]\n"
- "ld1 {v14.b}[4], [a_ptr7]\n"
+ "ld1 {v1.b}[4], [a_ptr1]\n"
+ "ld1 {v2.b}[4], [a_ptr2]\n"
+ "ld1 {v3.b}[4], [a_ptr3]\n"
+ "ld1 {v4.b}[4], [a_ptr4]\n"
+ "ld1 {v5.b}[4], [a_ptr5]\n"
+ "ld1 {v6.b}[4], [a_ptr6]\n"
+ "ld1 {v7.b}[4], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[2], [a_ptr1], #2\n"
- "ld1 {v4.h}[2], [a_ptr2], #2\n"
- "ld1 {v6.h}[2], [a_ptr3], #2\n"
- "ld1 {v8.h}[2], [a_ptr4], #2\n"
- "ld1 {v10.h}[2], [a_ptr5], #2\n"
- "ld1 {v12.h}[2], [a_ptr6], #2\n"
- "ld1 {v14.h}[2], [a_ptr7], #2\n"
+ "ld1 {v1.h}[2], [a_ptr1], #2\n"
+ "ld1 {v2.h}[2], [a_ptr2], #2\n"
+ "ld1 {v3.h}[2], [a_ptr3], #2\n"
+ "ld1 {v4.h}[2], [a_ptr4], #2\n"
+ "ld1 {v5.h}[2], [a_ptr5], #2\n"
+ "ld1 {v6.h}[2], [a_ptr6], #2\n"
+ "ld1 {v7.h}[2], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[6], [%[a_ptr0]]\n"
- "ld1 {v2.b}[6], [a_ptr1]\n"
- "ld1 {v4.b}[6], [a_ptr2]\n"
- "ld1 {v6.b}[6], [a_ptr3]\n"
- "ld1 {v8.b}[6], [a_ptr4]\n"
- "ld1 {v10.b}[6], [a_ptr5]\n"
- "ld1 {v12.b}[6], [a_ptr6]\n"
- "ld1 {v14.b}[6], [a_ptr7]\n"
+ "ld1 {v1.b}[6], [a_ptr1]\n"
+ "ld1 {v2.b}[6], [a_ptr2]\n"
+ "ld1 {v3.b}[6], [a_ptr3]\n"
+ "ld1 {v4.b}[6], [a_ptr4]\n"
+ "ld1 {v5.b}[6], [a_ptr5]\n"
+ "ld1 {v6.b}[6], [a_ptr6]\n"
+ "ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -398,117 +423,141 @@
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q25, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -588,56 +637,56 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v0.s}[2], [%[a_ptr0]]\n"
- "ld1 {v2.s}[2], [a_ptr1]\n"
- "ld1 {v4.s}[2], [a_ptr2]\n"
- "ld1 {v6.s}[2], [a_ptr3]\n"
- "ld1 {v8.s}[2], [a_ptr4]\n"
- "ld1 {v10.s}[2], [a_ptr5]\n"
- "ld1 {v12.s}[2], [a_ptr6]\n"
- "ld1 {v14.s}[2], [a_ptr7]\n"
+ "ld1 {v1.s}[2], [a_ptr1]\n"
+ "ld1 {v2.s}[2], [a_ptr2]\n"
+ "ld1 {v3.s}[2], [a_ptr3]\n"
+ "ld1 {v4.s}[2], [a_ptr4]\n"
+ "ld1 {v5.s}[2], [a_ptr5]\n"
+ "ld1 {v6.s}[2], [a_ptr6]\n"
+ "ld1 {v7.s}[2], [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[8], [%[a_ptr0]]\n"
- "ld1 {v2.b}[8], [a_ptr1]\n"
- "ld1 {v4.b}[8], [a_ptr2]\n"
- "ld1 {v6.b}[8], [a_ptr3]\n"
- "ld1 {v8.b}[8], [a_ptr4]\n"
- "ld1 {v10.b}[8], [a_ptr5]\n"
- "ld1 {v12.b}[8], [a_ptr6]\n"
- "ld1 {v14.b}[8], [a_ptr7]\n"
+ "ld1 {v1.b}[8], [a_ptr1]\n"
+ "ld1 {v2.b}[8], [a_ptr2]\n"
+ "ld1 {v3.b}[8], [a_ptr3]\n"
+ "ld1 {v4.b}[8], [a_ptr4]\n"
+ "ld1 {v5.b}[8], [a_ptr5]\n"
+ "ld1 {v6.b}[8], [a_ptr6]\n"
+ "ld1 {v7.b}[8], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[4], [a_ptr1], #2\n"
- "ld1 {v4.h}[4], [a_ptr2], #2\n"
- "ld1 {v6.h}[4], [a_ptr3], #2\n"
- "ld1 {v8.h}[4], [a_ptr4], #2\n"
- "ld1 {v10.h}[4], [a_ptr5], #2\n"
- "ld1 {v12.h}[4], [a_ptr6], #2\n"
- "ld1 {v14.h}[4], [a_ptr7], #2\n"
+ "ld1 {v1.h}[4], [a_ptr1], #2\n"
+ "ld1 {v2.h}[4], [a_ptr2], #2\n"
+ "ld1 {v3.h}[4], [a_ptr3], #2\n"
+ "ld1 {v4.h}[4], [a_ptr4], #2\n"
+ "ld1 {v5.h}[4], [a_ptr5], #2\n"
+ "ld1 {v6.h}[4], [a_ptr6], #2\n"
+ "ld1 {v7.h}[4], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[10], [%[a_ptr0]]\n"
- "ld1 {v2.b}[10], [a_ptr1]\n"
- "ld1 {v4.b}[10], [a_ptr2]\n"
- "ld1 {v6.b}[10], [a_ptr3]\n"
- "ld1 {v8.b}[10], [a_ptr4]\n"
- "ld1 {v10.b}[10], [a_ptr5]\n"
- "ld1 {v12.b}[10], [a_ptr6]\n"
- "ld1 {v14.b}[10], [a_ptr7]\n"
+ "ld1 {v1.b}[10], [a_ptr1]\n"
+ "ld1 {v2.b}[10], [a_ptr2]\n"
+ "ld1 {v3.b}[10], [a_ptr3]\n"
+ "ld1 {v4.b}[10], [a_ptr4]\n"
+ "ld1 {v5.b}[10], [a_ptr5]\n"
+ "ld1 {v6.b}[10], [a_ptr6]\n"
+ "ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -655,144 +704,168 @@
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q25, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -873,63 +946,63 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]]\n"
- "ldr q2, [a_ptr1]\n"
- "ldr q4, [a_ptr2]\n"
- "ldr q6, [a_ptr3]\n"
- "ldr q8, [a_ptr4]\n"
- "ldr q10, [a_ptr5]\n"
- "ldr q12, [a_ptr6]\n"
- "ldr q14, [a_ptr7]\n"
+ "ldr q1, [a_ptr1]\n"
+ "ldr q2, [a_ptr2]\n"
+ "ldr q3, [a_ptr3]\n"
+ "ldr q4, [a_ptr4]\n"
+ "ldr q5, [a_ptr5]\n"
+ "ldr q6, [a_ptr6]\n"
+ "ldr q7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"ld1 {v0.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v2.s}[2], [a_ptr1], #4\n"
- "ld1 {v4.s}[2], [a_ptr2], #4\n"
- "ld1 {v6.s}[2], [a_ptr3], #4\n"
- "ld1 {v8.s}[2], [a_ptr4], #4\n"
- "ld1 {v10.s}[2], [a_ptr5], #4\n"
- "ld1 {v12.s}[2], [a_ptr6], #4\n"
- "ld1 {v14.s}[2], [a_ptr7], #4\n"
+ "ld1 {v1.s}[2], [a_ptr1], #4\n"
+ "ld1 {v2.s}[2], [a_ptr2], #4\n"
+ "ld1 {v3.s}[2], [a_ptr3], #4\n"
+ "ld1 {v4.s}[2], [a_ptr4], #4\n"
+ "ld1 {v5.s}[2], [a_ptr5], #4\n"
+ "ld1 {v6.s}[2], [a_ptr6], #4\n"
+ "ld1 {v7.s}[2], [a_ptr7], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[12], [%[a_ptr0]]\n"
- "ld1 {v2.b}[12], [a_ptr1]\n"
- "ld1 {v4.b}[12], [a_ptr2]\n"
- "ld1 {v6.b}[12], [a_ptr3]\n"
- "ld1 {v8.b}[12], [a_ptr4]\n"
- "ld1 {v10.b}[12], [a_ptr5]\n"
- "ld1 {v12.b}[12], [a_ptr6]\n"
- "ld1 {v14.b}[12], [a_ptr7]\n"
+ "ld1 {v1.b}[12], [a_ptr1]\n"
+ "ld1 {v2.b}[12], [a_ptr2]\n"
+ "ld1 {v3.b}[12], [a_ptr3]\n"
+ "ld1 {v4.b}[12], [a_ptr4]\n"
+ "ld1 {v5.b}[12], [a_ptr5]\n"
+ "ld1 {v6.b}[12], [a_ptr6]\n"
+ "ld1 {v7.b}[12], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[6], [a_ptr1], #2\n"
- "ld1 {v4.h}[6], [a_ptr2], #2\n"
- "ld1 {v6.h}[6], [a_ptr3], #2\n"
- "ld1 {v8.h}[6], [a_ptr4], #2\n"
- "ld1 {v10.h}[6], [a_ptr5], #2\n"
- "ld1 {v12.h}[6], [a_ptr6], #2\n"
- "ld1 {v14.h}[6], [a_ptr7], #2\n"
+ "ld1 {v1.h}[6], [a_ptr1], #2\n"
+ "ld1 {v2.h}[6], [a_ptr2], #2\n"
+ "ld1 {v3.h}[6], [a_ptr3], #2\n"
+ "ld1 {v4.h}[6], [a_ptr4], #2\n"
+ "ld1 {v5.h}[6], [a_ptr5], #2\n"
+ "ld1 {v6.h}[6], [a_ptr6], #2\n"
+ "ld1 {v7.h}[6], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[14], [%[a_ptr0]]\n"
- "ld1 {v2.b}[14], [a_ptr1]\n"
- "ld1 {v4.b}[14], [a_ptr2]\n"
- "ld1 {v6.b}[14], [a_ptr3]\n"
- "ld1 {v8.b}[14], [a_ptr4]\n"
- "ld1 {v10.b}[14], [a_ptr5]\n"
- "ld1 {v12.b}[14], [a_ptr6]\n"
- "ld1 {v14.b}[14], [a_ptr7]\n"
+ "ld1 {v1.b}[14], [a_ptr1]\n"
+ "ld1 {v2.b}[14], [a_ptr2]\n"
+ "ld1 {v3.b}[14], [a_ptr3]\n"
+ "ld1 {v4.b}[14], [a_ptr4]\n"
+ "ld1 {v5.b}[14], [a_ptr5]\n"
+ "ld1 {v6.b}[14], [a_ptr6]\n"
+ "ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -947,41 +1020,41 @@
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -989,129 +1062,153 @@
"ldr q18, [%[b_ptr0], #0x20]\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q25, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1258,50 +1355,50 @@
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1310,146 +1407,170 @@
"ldr q19, [%[b_ptr0], #0x30]\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q25, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1604,59 +1725,59 @@
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1666,163 +1787,187 @@
"ldr q20, [%[b_ptr0], #0x40]\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ldr q21, [%[b_ptr0], #0x50]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q25, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ldr q21, [%[b_ptr0], #0x50]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1977,68 +2122,68 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2049,180 +2194,204 @@
"ldr q21, [%[b_ptr0], #0x50]\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ldr q22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q25, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"ldr q21, [%[b_ptr0], #0x50]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ldr q22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2386,77 +2555,77 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"ldr q23, [%[b_ptr0], #0x70]\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2468,197 +2637,221 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"b.eq 7f\n"
"8:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
"ldr q23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "str q25, [c_ptr1], #0x10\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q25, [c_ptr1]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
"ldr q21, [%[b_ptr0], #0x50]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
"ldr q22, [%[b_ptr0], #0x60]\n"
- ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
"b.ne 8b\n"
"7:\n"
- "str q24, [%[c_ptr0]], #0x10\n"
+ "str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
"ldr q23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- "str q25, [c_ptr1], #0x10\n"
+ "str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2], #0x10\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "str q27, [c_ptr3], #0x10\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "str q28, [c_ptr4], #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "str q29, [c_ptr5], #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "str q30, [c_ptr6], #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "str q31, [c_ptr7], #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp
index adeebf9..1110924 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 ARM Limited.
+ * Copyright (c) 2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -31,8 +31,8 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
-void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
class smallK_hybrid_u8u32_dot_4x6
{
@@ -40,7 +40,7 @@
typedef uint8_t operand_type;
typedef uint32_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+ typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -53,11 +53,26 @@
return 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {};
// Default to the generic kernel
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp
index 8bf5deb..ceb4a3b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp
@@ -25,14 +25,15 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- UNUSED(beta);
+void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -97,140 +98,155 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr s2, [%[a_ptr0]]\n"
- "ldr s6, [a_ptr1]\n"
- "ldr s10, [a_ptr2]\n"
- "ldr s14, [a_ptr3]\n"
- "ldr s18, [a_ptr4]\n"
- "ldr s22, [a_ptr5]\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
- "ldr b6, [a_ptr1]\n"
- "ldr b10, [a_ptr2]\n"
- "ldr b14, [a_ptr3]\n"
- "ldr b18, [a_ptr4]\n"
- "ldr b22, [a_ptr5]\n"
+ "ldr b5, [a_ptr1]\n"
+ "ldr b8, [a_ptr2]\n"
+ "ldr b11, [a_ptr3]\n"
+ "ldr b14, [a_ptr4]\n"
+ "ldr b17, [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ldr h2, [%[a_ptr0]], #0x2\n"
- "ldr h6, [a_ptr1], #0x2\n"
- "ldr h10, [a_ptr2], #0x2\n"
- "ldr h14, [a_ptr3], #0x2\n"
- "ldr h18, [a_ptr4], #0x2\n"
- "ldr h22, [a_ptr5], #0x2\n"
+ "ldr h5, [a_ptr1], #0x2\n"
+ "ldr h8, [a_ptr2], #0x2\n"
+ "ldr h11, [a_ptr3], #0x2\n"
+ "ldr h14, [a_ptr4], #0x2\n"
+ "ldr h17, [a_ptr5], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[2], [%[a_ptr0]]\n"
- "ld1 {v6.b}[2], [a_ptr1]\n"
- "ld1 {v10.b}[2], [a_ptr2]\n"
- "ld1 {v14.b}[2], [a_ptr3]\n"
- "ld1 {v18.b}[2], [a_ptr4]\n"
- "ld1 {v22.b}[2], [a_ptr5]\n"
+ "ld1 {v5.b}[2], [a_ptr1]\n"
+ "ld1 {v8.b}[2], [a_ptr2]\n"
+ "ld1 {v11.b}[2], [a_ptr3]\n"
+ "ld1 {v14.b}[2], [a_ptr4]\n"
+ "ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
"ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
@@ -238,114 +254,111 @@
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
@@ -354,83 +367,74 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -511,369 +515,372 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
- "ldr d6, [a_ptr1]\n"
- "ldr d10, [a_ptr2]\n"
- "ldr d14, [a_ptr3]\n"
- "ldr d18, [a_ptr4]\n"
- "ldr d22, [a_ptr5]\n"
+ "ldr d5, [a_ptr1]\n"
+ "ldr d8, [a_ptr2]\n"
+ "ldr d11, [a_ptr3]\n"
+ "ldr d14, [a_ptr4]\n"
+ "ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
- "ldr s6, [a_ptr1], #0x4\n"
- "ldr s10, [a_ptr2], #0x4\n"
- "ldr s14, [a_ptr3], #0x4\n"
- "ldr s18, [a_ptr4], #0x4\n"
- "ldr s22, [a_ptr5], #0x4\n"
+ "ldr s5, [a_ptr1], #0x4\n"
+ "ldr s8, [a_ptr2], #0x4\n"
+ "ldr s11, [a_ptr3], #0x4\n"
+ "ldr s14, [a_ptr4], #0x4\n"
+ "ldr s17, [a_ptr5], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
- "ld1 {v6.b}[4], [a_ptr1]\n"
- "ld1 {v10.b}[4], [a_ptr2]\n"
- "ld1 {v14.b}[4], [a_ptr3]\n"
- "ld1 {v18.b}[4], [a_ptr4]\n"
- "ld1 {v22.b}[4], [a_ptr5]\n"
+ "ld1 {v5.b}[4], [a_ptr1]\n"
+ "ld1 {v8.b}[4], [a_ptr2]\n"
+ "ld1 {v11.b}[4], [a_ptr3]\n"
+ "ld1 {v14.b}[4], [a_ptr4]\n"
+ "ld1 {v17.b}[4], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[2], [a_ptr1], #2\n"
- "ld1 {v10.h}[2], [a_ptr2], #2\n"
- "ld1 {v14.h}[2], [a_ptr3], #2\n"
- "ld1 {v18.h}[2], [a_ptr4], #2\n"
- "ld1 {v22.h}[2], [a_ptr5], #2\n"
+ "ld1 {v5.h}[2], [a_ptr1], #2\n"
+ "ld1 {v8.h}[2], [a_ptr2], #2\n"
+ "ld1 {v11.h}[2], [a_ptr3], #2\n"
+ "ld1 {v14.h}[2], [a_ptr4], #2\n"
+ "ld1 {v17.h}[2], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[6], [%[a_ptr0]]\n"
- "ld1 {v6.b}[6], [a_ptr1]\n"
- "ld1 {v10.b}[6], [a_ptr2]\n"
- "ld1 {v14.b}[6], [a_ptr3]\n"
- "ld1 {v18.b}[6], [a_ptr4]\n"
- "ld1 {v22.b}[6], [a_ptr5]\n"
+ "ld1 {v5.b}[6], [a_ptr1]\n"
+ "ld1 {v8.b}[6], [a_ptr2]\n"
+ "ld1 {v11.b}[6], [a_ptr3]\n"
+ "ld1 {v14.b}[6], [a_ptr4]\n"
+ "ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v23.d[1], temploadreg3\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -954,161 +961,175 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr q21, [a_ptr5], #0x10\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v2.s}[2], [%[a_ptr0]]\n"
- "ld1 {v6.s}[2], [a_ptr1]\n"
- "ld1 {v10.s}[2], [a_ptr2]\n"
- "ld1 {v14.s}[2], [a_ptr3]\n"
- "ld1 {v18.s}[2], [a_ptr4]\n"
- "ld1 {v22.s}[2], [a_ptr5]\n"
+ "ld1 {v5.s}[2], [a_ptr1]\n"
+ "ld1 {v8.s}[2], [a_ptr2]\n"
+ "ld1 {v11.s}[2], [a_ptr3]\n"
+ "ld1 {v14.s}[2], [a_ptr4]\n"
+ "ld1 {v17.s}[2], [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[8], [%[a_ptr0]]\n"
- "ld1 {v6.b}[8], [a_ptr1]\n"
- "ld1 {v10.b}[8], [a_ptr2]\n"
- "ld1 {v14.b}[8], [a_ptr3]\n"
- "ld1 {v18.b}[8], [a_ptr4]\n"
- "ld1 {v22.b}[8], [a_ptr5]\n"
+ "ld1 {v5.b}[8], [a_ptr1]\n"
+ "ld1 {v8.b}[8], [a_ptr2]\n"
+ "ld1 {v11.b}[8], [a_ptr3]\n"
+ "ld1 {v14.b}[8], [a_ptr4]\n"
+ "ld1 {v17.b}[8], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[4], [a_ptr1], #2\n"
- "ld1 {v10.h}[4], [a_ptr2], #2\n"
- "ld1 {v14.h}[4], [a_ptr3], #2\n"
- "ld1 {v18.h}[4], [a_ptr4], #2\n"
- "ld1 {v22.h}[4], [a_ptr5], #2\n"
+ "ld1 {v5.h}[4], [a_ptr1], #2\n"
+ "ld1 {v8.h}[4], [a_ptr2], #2\n"
+ "ld1 {v11.h}[4], [a_ptr3], #2\n"
+ "ld1 {v14.h}[4], [a_ptr4], #2\n"
+ "ld1 {v17.h}[4], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[10], [%[a_ptr0]]\n"
- "ld1 {v6.b}[10], [a_ptr1]\n"
- "ld1 {v10.b}[10], [a_ptr2]\n"
- "ld1 {v14.b}[10], [a_ptr3]\n"
- "ld1 {v18.b}[10], [a_ptr4]\n"
- "ld1 {v22.b}[10], [a_ptr5]\n"
+ "ld1 {v5.b}[10], [a_ptr1]\n"
+ "ld1 {v8.b}[10], [a_ptr2]\n"
+ "ld1 {v11.b}[10], [a_ptr3]\n"
+ "ld1 {v14.b}[10], [a_ptr4]\n"
+ "ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
"ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
@@ -1116,132 +1137,128 @@
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ins v23.d[1], temploadreg3\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
@@ -1251,98 +1268,88 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1423,174 +1430,188 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
- "ldr q6, [a_ptr1]\n"
- "ldr q10, [a_ptr2]\n"
- "ldr q14, [a_ptr3]\n"
- "ldr q18, [a_ptr4]\n"
- "ldr q22, [a_ptr5]\n"
+ "ldr q5, [a_ptr1]\n"
+ "ldr q8, [a_ptr2]\n"
+ "ldr q11, [a_ptr3]\n"
+ "ldr q14, [a_ptr4]\n"
+ "ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"ld1 {v2.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v6.s}[2], [a_ptr1], #4\n"
- "ld1 {v10.s}[2], [a_ptr2], #4\n"
- "ld1 {v14.s}[2], [a_ptr3], #4\n"
- "ld1 {v18.s}[2], [a_ptr4], #4\n"
- "ld1 {v22.s}[2], [a_ptr5], #4\n"
+ "ld1 {v5.s}[2], [a_ptr1], #4\n"
+ "ld1 {v8.s}[2], [a_ptr2], #4\n"
+ "ld1 {v11.s}[2], [a_ptr3], #4\n"
+ "ld1 {v14.s}[2], [a_ptr4], #4\n"
+ "ld1 {v17.s}[2], [a_ptr5], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
- "ld1 {v6.b}[12], [a_ptr1]\n"
- "ld1 {v10.b}[12], [a_ptr2]\n"
- "ld1 {v14.b}[12], [a_ptr3]\n"
- "ld1 {v18.b}[12], [a_ptr4]\n"
- "ld1 {v22.b}[12], [a_ptr5]\n"
+ "ld1 {v5.b}[12], [a_ptr1]\n"
+ "ld1 {v8.b}[12], [a_ptr2]\n"
+ "ld1 {v11.b}[12], [a_ptr3]\n"
+ "ld1 {v14.b}[12], [a_ptr4]\n"
+ "ld1 {v17.b}[12], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[6], [a_ptr1], #2\n"
- "ld1 {v10.h}[6], [a_ptr2], #2\n"
- "ld1 {v14.h}[6], [a_ptr3], #2\n"
- "ld1 {v18.h}[6], [a_ptr4], #2\n"
- "ld1 {v22.h}[6], [a_ptr5], #2\n"
+ "ld1 {v5.h}[6], [a_ptr1], #2\n"
+ "ld1 {v8.h}[6], [a_ptr2], #2\n"
+ "ld1 {v11.h}[6], [a_ptr3], #2\n"
+ "ld1 {v14.h}[6], [a_ptr4], #2\n"
+ "ld1 {v17.h}[6], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[14], [%[a_ptr0]]\n"
- "ld1 {v6.b}[14], [a_ptr1]\n"
- "ld1 {v10.b}[14], [a_ptr2]\n"
- "ld1 {v14.b}[14], [a_ptr3]\n"
- "ld1 {v18.b}[14], [a_ptr4]\n"
- "ld1 {v22.b}[14], [a_ptr5]\n"
+ "ld1 {v5.b}[14], [a_ptr1]\n"
+ "ld1 {v8.b}[14], [a_ptr2]\n"
+ "ld1 {v11.b}[14], [a_ptr3]\n"
+ "ld1 {v14.b}[14], [a_ptr4]\n"
+ "ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
"cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
+ "ldr d18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ "ins v19.d[1], temploadreg3\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ "ins v20.d[1], temploadreg0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ins v22.d[1], temploadreg2\n"
+ "ins v23.d[1], temploadreg3\n"
"ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
@@ -1598,141 +1619,137 @@
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
"ins v25.d[1], temploadreg1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "ins v21.d[1], temploadreg1\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
@@ -1742,105 +1759,95 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1986,104 +1993,104 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2105,142 +2112,142 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"ins v24.d[1], temploadreg0\n"
@@ -2252,113 +2259,113 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -2510,111 +2517,111 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2635,152 +2642,152 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
@@ -2792,120 +2799,120 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -3057,119 +3064,119 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -3191,161 +3198,161 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"ins v24.d[1], temploadreg0\n"
@@ -3357,128 +3364,128 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -3637,126 +3644,126 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr d24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -3777,171 +3784,171 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
- ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
- ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
@@ -3953,135 +3960,135 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp
index 1f279af..4eecee4 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp
@@ -25,14 +25,15 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- UNUSED(beta);
+void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -93,318 +94,309 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr s2, [%[a_ptr0]]\n"
- "ldr s6, [a_ptr1]\n"
- "ldr s10, [a_ptr2]\n"
- "ldr s14, [a_ptr3]\n"
- "ldr s18, [a_ptr4]\n"
- "ldr s22, [a_ptr5]\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
- "ldr b6, [a_ptr1]\n"
- "ldr b10, [a_ptr2]\n"
- "ldr b14, [a_ptr3]\n"
- "ldr b18, [a_ptr4]\n"
- "ldr b22, [a_ptr5]\n"
+ "ldr b5, [a_ptr1]\n"
+ "ldr b8, [a_ptr2]\n"
+ "ldr b11, [a_ptr3]\n"
+ "ldr b14, [a_ptr4]\n"
+ "ldr b17, [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ldr h2, [%[a_ptr0]], #0x2\n"
- "ldr h6, [a_ptr1], #0x2\n"
- "ldr h10, [a_ptr2], #0x2\n"
- "ldr h14, [a_ptr3], #0x2\n"
- "ldr h18, [a_ptr4], #0x2\n"
- "ldr h22, [a_ptr5], #0x2\n"
+ "ldr h5, [a_ptr1], #0x2\n"
+ "ldr h8, [a_ptr2], #0x2\n"
+ "ldr h11, [a_ptr3], #0x2\n"
+ "ldr h14, [a_ptr4], #0x2\n"
+ "ldr h17, [a_ptr5], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[2], [%[a_ptr0]]\n"
- "ld1 {v6.b}[2], [a_ptr1]\n"
- "ld1 {v10.b}[2], [a_ptr2]\n"
- "ld1 {v14.b}[2], [a_ptr3]\n"
- "ld1 {v18.b}[2], [a_ptr4]\n"
- "ld1 {v22.b}[2], [a_ptr5]\n"
+ "ld1 {v5.b}[2], [a_ptr1]\n"
+ "ld1 {v8.b}[2], [a_ptr2]\n"
+ "ld1 {v11.b}[2], [a_ptr3]\n"
+ "ld1 {v14.b}[2], [a_ptr4]\n"
+ "ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
"str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -477,345 +469,336 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
- "ldr d6, [a_ptr1]\n"
- "ldr d10, [a_ptr2]\n"
- "ldr d14, [a_ptr3]\n"
- "ldr d18, [a_ptr4]\n"
- "ldr d22, [a_ptr5]\n"
+ "ldr d5, [a_ptr1]\n"
+ "ldr d8, [a_ptr2]\n"
+ "ldr d11, [a_ptr3]\n"
+ "ldr d14, [a_ptr4]\n"
+ "ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
- "ldr s6, [a_ptr1], #0x4\n"
- "ldr s10, [a_ptr2], #0x4\n"
- "ldr s14, [a_ptr3], #0x4\n"
- "ldr s18, [a_ptr4], #0x4\n"
- "ldr s22, [a_ptr5], #0x4\n"
+ "ldr s5, [a_ptr1], #0x4\n"
+ "ldr s8, [a_ptr2], #0x4\n"
+ "ldr s11, [a_ptr3], #0x4\n"
+ "ldr s14, [a_ptr4], #0x4\n"
+ "ldr s17, [a_ptr5], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
- "ld1 {v6.b}[4], [a_ptr1]\n"
- "ld1 {v10.b}[4], [a_ptr2]\n"
- "ld1 {v14.b}[4], [a_ptr3]\n"
- "ld1 {v18.b}[4], [a_ptr4]\n"
- "ld1 {v22.b}[4], [a_ptr5]\n"
+ "ld1 {v5.b}[4], [a_ptr1]\n"
+ "ld1 {v8.b}[4], [a_ptr2]\n"
+ "ld1 {v11.b}[4], [a_ptr3]\n"
+ "ld1 {v14.b}[4], [a_ptr4]\n"
+ "ld1 {v17.b}[4], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[2], [a_ptr1], #2\n"
- "ld1 {v10.h}[2], [a_ptr2], #2\n"
- "ld1 {v14.h}[2], [a_ptr3], #2\n"
- "ld1 {v18.h}[2], [a_ptr4], #2\n"
- "ld1 {v22.h}[2], [a_ptr5], #2\n"
+ "ld1 {v5.h}[2], [a_ptr1], #2\n"
+ "ld1 {v8.h}[2], [a_ptr2], #2\n"
+ "ld1 {v11.h}[2], [a_ptr3], #2\n"
+ "ld1 {v14.h}[2], [a_ptr4], #2\n"
+ "ld1 {v17.h}[2], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[6], [%[a_ptr0]]\n"
- "ld1 {v6.b}[6], [a_ptr1]\n"
- "ld1 {v10.b}[6], [a_ptr2]\n"
- "ld1 {v14.b}[6], [a_ptr3]\n"
- "ld1 {v18.b}[6], [a_ptr4]\n"
- "ld1 {v22.b}[6], [a_ptr5]\n"
+ "ld1 {v5.b}[6], [a_ptr1]\n"
+ "ld1 {v8.b}[6], [a_ptr2]\n"
+ "ld1 {v11.b}[6], [a_ptr3]\n"
+ "ld1 {v14.b}[6], [a_ptr4]\n"
+ "ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "subs %[loops], %[loops], #0x1\n"
"str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -888,369 +871,357 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr q21, [a_ptr5], #0x10\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v2.s}[2], [%[a_ptr0]]\n"
- "ld1 {v6.s}[2], [a_ptr1]\n"
- "ld1 {v10.s}[2], [a_ptr2]\n"
- "ld1 {v14.s}[2], [a_ptr3]\n"
- "ld1 {v18.s}[2], [a_ptr4]\n"
- "ld1 {v22.s}[2], [a_ptr5]\n"
+ "ld1 {v5.s}[2], [a_ptr1]\n"
+ "ld1 {v8.s}[2], [a_ptr2]\n"
+ "ld1 {v11.s}[2], [a_ptr3]\n"
+ "ld1 {v14.s}[2], [a_ptr4]\n"
+ "ld1 {v17.s}[2], [a_ptr5]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[8], [%[a_ptr0]]\n"
- "ld1 {v6.b}[8], [a_ptr1]\n"
- "ld1 {v10.b}[8], [a_ptr2]\n"
- "ld1 {v14.b}[8], [a_ptr3]\n"
- "ld1 {v18.b}[8], [a_ptr4]\n"
- "ld1 {v22.b}[8], [a_ptr5]\n"
+ "ld1 {v5.b}[8], [a_ptr1]\n"
+ "ld1 {v8.b}[8], [a_ptr2]\n"
+ "ld1 {v11.b}[8], [a_ptr3]\n"
+ "ld1 {v14.b}[8], [a_ptr4]\n"
+ "ld1 {v17.b}[8], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[4], [a_ptr1], #2\n"
- "ld1 {v10.h}[4], [a_ptr2], #2\n"
- "ld1 {v14.h}[4], [a_ptr3], #2\n"
- "ld1 {v18.h}[4], [a_ptr4], #2\n"
- "ld1 {v22.h}[4], [a_ptr5], #2\n"
+ "ld1 {v5.h}[4], [a_ptr1], #2\n"
+ "ld1 {v8.h}[4], [a_ptr2], #2\n"
+ "ld1 {v11.h}[4], [a_ptr3], #2\n"
+ "ld1 {v14.h}[4], [a_ptr4], #2\n"
+ "ld1 {v17.h}[4], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[10], [%[a_ptr0]]\n"
- "ld1 {v6.b}[10], [a_ptr1]\n"
- "ld1 {v10.b}[10], [a_ptr2]\n"
- "ld1 {v14.b}[10], [a_ptr3]\n"
- "ld1 {v18.b}[10], [a_ptr4]\n"
- "ld1 {v22.b}[10], [a_ptr5]\n"
+ "ld1 {v5.b}[10], [a_ptr1]\n"
+ "ld1 {v8.b}[10], [a_ptr2]\n"
+ "ld1 {v11.b}[10], [a_ptr3]\n"
+ "ld1 {v14.b}[10], [a_ptr4]\n"
+ "ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
"str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1323,396 +1294,384 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
- "ldr q4, [a_ptr1], #0x10\n"
- "ldr q8, [a_ptr2], #0x10\n"
- "ldr q12, [a_ptr3], #0x10\n"
- "ldr q16, [a_ptr4], #0x10\n"
- "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
"ldr q1, [%[a_ptr0]], #0x10\n"
- "ldr q5, [a_ptr1], #0x10\n"
- "ldr q9, [a_ptr2], #0x10\n"
- "ldr q13, [a_ptr3], #0x10\n"
- "ldr q17, [a_ptr4], #0x10\n"
- "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
- "ldr q6, [a_ptr1]\n"
- "ldr q10, [a_ptr2]\n"
- "ldr q14, [a_ptr3]\n"
- "ldr q18, [a_ptr4]\n"
- "ldr q22, [a_ptr5]\n"
+ "ldr q5, [a_ptr1]\n"
+ "ldr q8, [a_ptr2]\n"
+ "ldr q11, [a_ptr3]\n"
+ "ldr q14, [a_ptr4]\n"
+ "ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
- "ldr d6, [a_ptr1], #0x8\n"
- "ldr d10, [a_ptr2], #0x8\n"
- "ldr d14, [a_ptr3], #0x8\n"
- "ldr d18, [a_ptr4], #0x8\n"
- "ldr d22, [a_ptr5], #0x8\n"
+ "ldr d5, [a_ptr1], #0x8\n"
+ "ldr d8, [a_ptr2], #0x8\n"
+ "ldr d11, [a_ptr3], #0x8\n"
+ "ldr d14, [a_ptr4], #0x8\n"
+ "ldr d17, [a_ptr5], #0x8\n"
"ld1 {v2.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v6.s}[2], [a_ptr1], #4\n"
- "ld1 {v10.s}[2], [a_ptr2], #4\n"
- "ld1 {v14.s}[2], [a_ptr3], #4\n"
- "ld1 {v18.s}[2], [a_ptr4], #4\n"
- "ld1 {v22.s}[2], [a_ptr5], #4\n"
+ "ld1 {v5.s}[2], [a_ptr1], #4\n"
+ "ld1 {v8.s}[2], [a_ptr2], #4\n"
+ "ld1 {v11.s}[2], [a_ptr3], #4\n"
+ "ld1 {v14.s}[2], [a_ptr4], #4\n"
+ "ld1 {v17.s}[2], [a_ptr5], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
- "ld1 {v6.b}[12], [a_ptr1]\n"
- "ld1 {v10.b}[12], [a_ptr2]\n"
- "ld1 {v14.b}[12], [a_ptr3]\n"
- "ld1 {v18.b}[12], [a_ptr4]\n"
- "ld1 {v22.b}[12], [a_ptr5]\n"
+ "ld1 {v5.b}[12], [a_ptr1]\n"
+ "ld1 {v8.b}[12], [a_ptr2]\n"
+ "ld1 {v11.b}[12], [a_ptr3]\n"
+ "ld1 {v14.b}[12], [a_ptr4]\n"
+ "ld1 {v17.b}[12], [a_ptr5]\n"
"b 3f\n"
"4:\n"
"ld1 {v2.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v6.h}[6], [a_ptr1], #2\n"
- "ld1 {v10.h}[6], [a_ptr2], #2\n"
- "ld1 {v14.h}[6], [a_ptr3], #2\n"
- "ld1 {v18.h}[6], [a_ptr4], #2\n"
- "ld1 {v22.h}[6], [a_ptr5], #2\n"
+ "ld1 {v5.h}[6], [a_ptr1], #2\n"
+ "ld1 {v8.h}[6], [a_ptr2], #2\n"
+ "ld1 {v11.h}[6], [a_ptr3], #2\n"
+ "ld1 {v14.h}[6], [a_ptr4], #2\n"
+ "ld1 {v17.h}[6], [a_ptr5], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v2.b}[14], [%[a_ptr0]]\n"
- "ld1 {v6.b}[14], [a_ptr1]\n"
- "ld1 {v10.b}[14], [a_ptr2]\n"
- "ld1 {v14.b}[14], [a_ptr3]\n"
- "ld1 {v18.b}[14], [a_ptr4]\n"
- "ld1 {v22.b}[14], [a_ptr5]\n"
+ "ld1 {v5.b}[14], [a_ptr1]\n"
+ "ld1 {v8.b}[14], [a_ptr2]\n"
+ "ld1 {v11.b}[14], [a_ptr3]\n"
+ "ld1 {v14.b}[14], [a_ptr4]\n"
+ "ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
"movi v26.4s, #0\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"movi v31.4s, #0\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
"cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
+ "ldr q18, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "subs %[loops], %[loops], #0x1\n"
"str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
"movi v26.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1850,104 +1809,104 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1962,120 +1921,120 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
@@ -2086,113 +2045,113 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -2336,111 +2295,111 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2455,127 +2414,127 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
@@ -2586,120 +2545,120 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -2843,119 +2802,119 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2970,135 +2929,135 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
@@ -3109,128 +3068,128 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -3381,126 +3340,126 @@
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr q24, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -3515,142 +3474,142 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v27.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
@@ -3661,135 +3620,135 @@
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
- ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
- ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
- ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
- ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
- ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
- ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
- ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
- ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
- ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
- ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
- ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
- ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
- ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
- ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
- ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
- ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
- ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
- ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
- ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
- ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
- ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
- ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
- ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
- ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
- ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
- ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
- ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
- ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
- ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
- ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
- ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
- ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
- ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
- ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
- ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
- ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
- ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
- ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
- ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
- ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
- ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
- ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
- ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
- ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
- ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
- ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
- ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
- ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
- ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
- ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
- ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
- ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
- ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
- ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
- ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
- ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
- ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
- ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
"6:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp
index a260917..fef8909 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 ARM Limited.
+ * Copyright (c) 2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -31,8 +31,8 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
-void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
class smallK_hybrid_u8u32_dot_4x8
{
@@ -40,7 +40,7 @@
typedef uint8_t operand_type;
typedef uint32_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+ typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -53,11 +53,26 @@
return 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {};
// Default to the generic kernel
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp
index ec29698..f30f7e2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp
@@ -25,14 +25,15 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- UNUSED(beta);
+void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -114,47 +115,47 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr s0, [%[a_ptr0]]\n"
- "ldr s2, [a_ptr1]\n"
- "ldr s4, [a_ptr2]\n"
- "ldr s6, [a_ptr3]\n"
- "ldr s8, [a_ptr4]\n"
- "ldr s10, [a_ptr5]\n"
- "ldr s12, [a_ptr6]\n"
- "ldr s14, [a_ptr7]\n"
+ "ldr s1, [a_ptr1]\n"
+ "ldr s2, [a_ptr2]\n"
+ "ldr s3, [a_ptr3]\n"
+ "ldr s4, [a_ptr4]\n"
+ "ldr s5, [a_ptr5]\n"
+ "ldr s6, [a_ptr6]\n"
+ "ldr s7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b0, [%[a_ptr0]]\n"
- "ldr b2, [a_ptr1]\n"
- "ldr b4, [a_ptr2]\n"
- "ldr b6, [a_ptr3]\n"
- "ldr b8, [a_ptr4]\n"
- "ldr b10, [a_ptr5]\n"
- "ldr b12, [a_ptr6]\n"
- "ldr b14, [a_ptr7]\n"
+ "ldr b1, [a_ptr1]\n"
+ "ldr b2, [a_ptr2]\n"
+ "ldr b3, [a_ptr3]\n"
+ "ldr b4, [a_ptr4]\n"
+ "ldr b5, [a_ptr5]\n"
+ "ldr b6, [a_ptr6]\n"
+ "ldr b7, [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ldr h0, [%[a_ptr0]], #0x2\n"
- "ldr h2, [a_ptr1], #0x2\n"
- "ldr h4, [a_ptr2], #0x2\n"
- "ldr h6, [a_ptr3], #0x2\n"
- "ldr h8, [a_ptr4], #0x2\n"
- "ldr h10, [a_ptr5], #0x2\n"
- "ldr h12, [a_ptr6], #0x2\n"
- "ldr h14, [a_ptr7], #0x2\n"
+ "ldr h1, [a_ptr1], #0x2\n"
+ "ldr h2, [a_ptr2], #0x2\n"
+ "ldr h3, [a_ptr3], #0x2\n"
+ "ldr h4, [a_ptr4], #0x2\n"
+ "ldr h5, [a_ptr5], #0x2\n"
+ "ldr h6, [a_ptr6], #0x2\n"
+ "ldr h7, [a_ptr7], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v2.b}[2], [a_ptr1]\n"
- "ld1 {v4.b}[2], [a_ptr2]\n"
- "ld1 {v6.b}[2], [a_ptr3]\n"
- "ld1 {v8.b}[2], [a_ptr4]\n"
- "ld1 {v10.b}[2], [a_ptr5]\n"
- "ld1 {v12.b}[2], [a_ptr6]\n"
- "ld1 {v14.b}[2], [a_ptr7]\n"
+ "ld1 {v1.b}[2], [a_ptr1]\n"
+ "ld1 {v2.b}[2], [a_ptr2]\n"
+ "ld1 {v3.b}[2], [a_ptr3]\n"
+ "ld1 {v4.b}[2], [a_ptr4]\n"
+ "ld1 {v5.b}[2], [a_ptr5]\n"
+ "ld1 {v6.b}[2], [a_ptr6]\n"
+ "ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -172,14 +173,14 @@
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -197,33 +198,33 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
@@ -241,32 +242,32 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -372,55 +373,55 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr d0, [%[a_ptr0]]\n"
- "ldr d2, [a_ptr1]\n"
- "ldr d4, [a_ptr2]\n"
- "ldr d6, [a_ptr3]\n"
- "ldr d8, [a_ptr4]\n"
- "ldr d10, [a_ptr5]\n"
- "ldr d12, [a_ptr6]\n"
- "ldr d14, [a_ptr7]\n"
+ "ldr d1, [a_ptr1]\n"
+ "ldr d2, [a_ptr2]\n"
+ "ldr d3, [a_ptr3]\n"
+ "ldr d4, [a_ptr4]\n"
+ "ldr d5, [a_ptr5]\n"
+ "ldr d6, [a_ptr6]\n"
+ "ldr d7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr s0, [%[a_ptr0]], #0x4\n"
- "ldr s2, [a_ptr1], #0x4\n"
- "ldr s4, [a_ptr2], #0x4\n"
- "ldr s6, [a_ptr3], #0x4\n"
- "ldr s8, [a_ptr4], #0x4\n"
- "ldr s10, [a_ptr5], #0x4\n"
- "ldr s12, [a_ptr6], #0x4\n"
- "ldr s14, [a_ptr7], #0x4\n"
+ "ldr s1, [a_ptr1], #0x4\n"
+ "ldr s2, [a_ptr2], #0x4\n"
+ "ldr s3, [a_ptr3], #0x4\n"
+ "ldr s4, [a_ptr4], #0x4\n"
+ "ldr s5, [a_ptr5], #0x4\n"
+ "ldr s6, [a_ptr6], #0x4\n"
+ "ldr s7, [a_ptr7], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[4], [%[a_ptr0]]\n"
- "ld1 {v2.b}[4], [a_ptr1]\n"
- "ld1 {v4.b}[4], [a_ptr2]\n"
- "ld1 {v6.b}[4], [a_ptr3]\n"
- "ld1 {v8.b}[4], [a_ptr4]\n"
- "ld1 {v10.b}[4], [a_ptr5]\n"
- "ld1 {v12.b}[4], [a_ptr6]\n"
- "ld1 {v14.b}[4], [a_ptr7]\n"
+ "ld1 {v1.b}[4], [a_ptr1]\n"
+ "ld1 {v2.b}[4], [a_ptr2]\n"
+ "ld1 {v3.b}[4], [a_ptr3]\n"
+ "ld1 {v4.b}[4], [a_ptr4]\n"
+ "ld1 {v5.b}[4], [a_ptr5]\n"
+ "ld1 {v6.b}[4], [a_ptr6]\n"
+ "ld1 {v7.b}[4], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[2], [a_ptr1], #2\n"
- "ld1 {v4.h}[2], [a_ptr2], #2\n"
- "ld1 {v6.h}[2], [a_ptr3], #2\n"
- "ld1 {v8.h}[2], [a_ptr4], #2\n"
- "ld1 {v10.h}[2], [a_ptr5], #2\n"
- "ld1 {v12.h}[2], [a_ptr6], #2\n"
- "ld1 {v14.h}[2], [a_ptr7], #2\n"
+ "ld1 {v1.h}[2], [a_ptr1], #2\n"
+ "ld1 {v2.h}[2], [a_ptr2], #2\n"
+ "ld1 {v3.h}[2], [a_ptr3], #2\n"
+ "ld1 {v4.h}[2], [a_ptr4], #2\n"
+ "ld1 {v5.h}[2], [a_ptr5], #2\n"
+ "ld1 {v6.h}[2], [a_ptr6], #2\n"
+ "ld1 {v7.h}[2], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[6], [%[a_ptr0]]\n"
- "ld1 {v2.b}[6], [a_ptr1]\n"
- "ld1 {v4.b}[6], [a_ptr2]\n"
- "ld1 {v6.b}[6], [a_ptr3]\n"
- "ld1 {v8.b}[6], [a_ptr4]\n"
- "ld1 {v10.b}[6], [a_ptr5]\n"
- "ld1 {v12.b}[6], [a_ptr6]\n"
- "ld1 {v14.b}[6], [a_ptr7]\n"
+ "ld1 {v1.b}[6], [a_ptr1]\n"
+ "ld1 {v2.b}[6], [a_ptr2]\n"
+ "ld1 {v3.b}[6], [a_ptr3]\n"
+ "ld1 {v4.b}[6], [a_ptr4]\n"
+ "ld1 {v5.b}[6], [a_ptr5]\n"
+ "ld1 {v6.b}[6], [a_ptr6]\n"
+ "ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -438,23 +439,23 @@
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -475,49 +476,49 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
@@ -530,40 +531,40 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -668,56 +669,56 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v0.s}[2], [%[a_ptr0]]\n"
- "ld1 {v2.s}[2], [a_ptr1]\n"
- "ld1 {v4.s}[2], [a_ptr2]\n"
- "ld1 {v6.s}[2], [a_ptr3]\n"
- "ld1 {v8.s}[2], [a_ptr4]\n"
- "ld1 {v10.s}[2], [a_ptr5]\n"
- "ld1 {v12.s}[2], [a_ptr6]\n"
- "ld1 {v14.s}[2], [a_ptr7]\n"
+ "ld1 {v1.s}[2], [a_ptr1]\n"
+ "ld1 {v2.s}[2], [a_ptr2]\n"
+ "ld1 {v3.s}[2], [a_ptr3]\n"
+ "ld1 {v4.s}[2], [a_ptr4]\n"
+ "ld1 {v5.s}[2], [a_ptr5]\n"
+ "ld1 {v6.s}[2], [a_ptr6]\n"
+ "ld1 {v7.s}[2], [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[8], [%[a_ptr0]]\n"
- "ld1 {v2.b}[8], [a_ptr1]\n"
- "ld1 {v4.b}[8], [a_ptr2]\n"
- "ld1 {v6.b}[8], [a_ptr3]\n"
- "ld1 {v8.b}[8], [a_ptr4]\n"
- "ld1 {v10.b}[8], [a_ptr5]\n"
- "ld1 {v12.b}[8], [a_ptr6]\n"
- "ld1 {v14.b}[8], [a_ptr7]\n"
+ "ld1 {v1.b}[8], [a_ptr1]\n"
+ "ld1 {v2.b}[8], [a_ptr2]\n"
+ "ld1 {v3.b}[8], [a_ptr3]\n"
+ "ld1 {v4.b}[8], [a_ptr4]\n"
+ "ld1 {v5.b}[8], [a_ptr5]\n"
+ "ld1 {v6.b}[8], [a_ptr6]\n"
+ "ld1 {v7.b}[8], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[4], [a_ptr1], #2\n"
- "ld1 {v4.h}[4], [a_ptr2], #2\n"
- "ld1 {v6.h}[4], [a_ptr3], #2\n"
- "ld1 {v8.h}[4], [a_ptr4], #2\n"
- "ld1 {v10.h}[4], [a_ptr5], #2\n"
- "ld1 {v12.h}[4], [a_ptr6], #2\n"
- "ld1 {v14.h}[4], [a_ptr7], #2\n"
+ "ld1 {v1.h}[4], [a_ptr1], #2\n"
+ "ld1 {v2.h}[4], [a_ptr2], #2\n"
+ "ld1 {v3.h}[4], [a_ptr3], #2\n"
+ "ld1 {v4.h}[4], [a_ptr4], #2\n"
+ "ld1 {v5.h}[4], [a_ptr5], #2\n"
+ "ld1 {v6.h}[4], [a_ptr6], #2\n"
+ "ld1 {v7.h}[4], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[10], [%[a_ptr0]]\n"
- "ld1 {v2.b}[10], [a_ptr1]\n"
- "ld1 {v4.b}[10], [a_ptr2]\n"
- "ld1 {v6.b}[10], [a_ptr3]\n"
- "ld1 {v8.b}[10], [a_ptr4]\n"
- "ld1 {v10.b}[10], [a_ptr5]\n"
- "ld1 {v12.b}[10], [a_ptr6]\n"
- "ld1 {v14.b}[10], [a_ptr7]\n"
+ "ld1 {v1.b}[10], [a_ptr1]\n"
+ "ld1 {v2.b}[10], [a_ptr2]\n"
+ "ld1 {v3.b}[10], [a_ptr3]\n"
+ "ld1 {v4.b}[10], [a_ptr4]\n"
+ "ld1 {v5.b}[10], [a_ptr5]\n"
+ "ld1 {v6.b}[10], [a_ptr6]\n"
+ "ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -735,32 +736,32 @@
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -784,61 +785,61 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
"b.ne 8b\n"
@@ -850,48 +851,48 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -997,63 +998,63 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]]\n"
- "ldr q2, [a_ptr1]\n"
- "ldr q4, [a_ptr2]\n"
- "ldr q6, [a_ptr3]\n"
- "ldr q8, [a_ptr4]\n"
- "ldr q10, [a_ptr5]\n"
- "ldr q12, [a_ptr6]\n"
- "ldr q14, [a_ptr7]\n"
+ "ldr q1, [a_ptr1]\n"
+ "ldr q2, [a_ptr2]\n"
+ "ldr q3, [a_ptr3]\n"
+ "ldr q4, [a_ptr4]\n"
+ "ldr q5, [a_ptr5]\n"
+ "ldr q6, [a_ptr6]\n"
+ "ldr q7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"ld1 {v0.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v2.s}[2], [a_ptr1], #4\n"
- "ld1 {v4.s}[2], [a_ptr2], #4\n"
- "ld1 {v6.s}[2], [a_ptr3], #4\n"
- "ld1 {v8.s}[2], [a_ptr4], #4\n"
- "ld1 {v10.s}[2], [a_ptr5], #4\n"
- "ld1 {v12.s}[2], [a_ptr6], #4\n"
- "ld1 {v14.s}[2], [a_ptr7], #4\n"
+ "ld1 {v1.s}[2], [a_ptr1], #4\n"
+ "ld1 {v2.s}[2], [a_ptr2], #4\n"
+ "ld1 {v3.s}[2], [a_ptr3], #4\n"
+ "ld1 {v4.s}[2], [a_ptr4], #4\n"
+ "ld1 {v5.s}[2], [a_ptr5], #4\n"
+ "ld1 {v6.s}[2], [a_ptr6], #4\n"
+ "ld1 {v7.s}[2], [a_ptr7], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[12], [%[a_ptr0]]\n"
- "ld1 {v2.b}[12], [a_ptr1]\n"
- "ld1 {v4.b}[12], [a_ptr2]\n"
- "ld1 {v6.b}[12], [a_ptr3]\n"
- "ld1 {v8.b}[12], [a_ptr4]\n"
- "ld1 {v10.b}[12], [a_ptr5]\n"
- "ld1 {v12.b}[12], [a_ptr6]\n"
- "ld1 {v14.b}[12], [a_ptr7]\n"
+ "ld1 {v1.b}[12], [a_ptr1]\n"
+ "ld1 {v2.b}[12], [a_ptr2]\n"
+ "ld1 {v3.b}[12], [a_ptr3]\n"
+ "ld1 {v4.b}[12], [a_ptr4]\n"
+ "ld1 {v5.b}[12], [a_ptr5]\n"
+ "ld1 {v6.b}[12], [a_ptr6]\n"
+ "ld1 {v7.b}[12], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[6], [a_ptr1], #2\n"
- "ld1 {v4.h}[6], [a_ptr2], #2\n"
- "ld1 {v6.h}[6], [a_ptr3], #2\n"
- "ld1 {v8.h}[6], [a_ptr4], #2\n"
- "ld1 {v10.h}[6], [a_ptr5], #2\n"
- "ld1 {v12.h}[6], [a_ptr6], #2\n"
- "ld1 {v14.h}[6], [a_ptr7], #2\n"
+ "ld1 {v1.h}[6], [a_ptr1], #2\n"
+ "ld1 {v2.h}[6], [a_ptr2], #2\n"
+ "ld1 {v3.h}[6], [a_ptr3], #2\n"
+ "ld1 {v4.h}[6], [a_ptr4], #2\n"
+ "ld1 {v5.h}[6], [a_ptr5], #2\n"
+ "ld1 {v6.h}[6], [a_ptr6], #2\n"
+ "ld1 {v7.h}[6], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[14], [%[a_ptr0]]\n"
- "ld1 {v2.b}[14], [a_ptr1]\n"
- "ld1 {v4.b}[14], [a_ptr2]\n"
- "ld1 {v6.b}[14], [a_ptr3]\n"
- "ld1 {v8.b}[14], [a_ptr4]\n"
- "ld1 {v10.b}[14], [a_ptr5]\n"
- "ld1 {v12.b}[14], [a_ptr6]\n"
- "ld1 {v14.b}[14], [a_ptr7]\n"
+ "ld1 {v1.b}[14], [a_ptr1]\n"
+ "ld1 {v2.b}[14], [a_ptr2]\n"
+ "ld1 {v3.b}[14], [a_ptr3]\n"
+ "ld1 {v4.b}[14], [a_ptr4]\n"
+ "ld1 {v5.b}[14], [a_ptr5]\n"
+ "ld1 {v6.b}[14], [a_ptr6]\n"
+ "ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -1071,41 +1072,41 @@
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1132,72 +1133,72 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
"b.ne 8b\n"
@@ -1209,56 +1210,56 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1430,50 +1431,50 @@
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1503,83 +1504,83 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
"b.ne 8b\n"
@@ -1591,64 +1592,64 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1828,59 +1829,59 @@
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1913,94 +1914,94 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
"ins v20.d[1], temploadreg0\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"ldr d21, [%[b_ptr0], #0x50]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
"b.ne 8b\n"
@@ -2012,72 +2013,72 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -2257,68 +2258,68 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2354,105 +2355,105 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
"ins v20.d[1], temploadreg0\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"ldr d21, [%[b_ptr0], #0x50]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
"ins v21.d[1], temploadreg1\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"ldr d22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
"b.ne 8b\n"
@@ -2464,80 +2465,80 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -2726,77 +2727,77 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"ldr q23, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr d16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2835,116 +2836,116 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr d16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"ins v16.d[1], temploadreg0\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr d17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
"ins v17.d[1], temploadreg1\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr d18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
"ins v18.d[1], temploadreg2\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
"ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr d19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
"ins v19.d[1], temploadreg3\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
"ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"ldr d20, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
"ins v20.d[1], temploadreg0\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"ldr d21, [%[b_ptr0], #0x50]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
"ins v21.d[1], temploadreg1\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"ldr d22, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
"ins v22.d[1], temploadreg2\n"
- ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
"ldr d23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
@@ -2956,88 +2957,88 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp
index 46ca013..16c1d17 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp
@@ -25,14 +25,15 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- UNUSED(beta);
+void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -110,47 +111,47 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr s0, [%[a_ptr0]]\n"
- "ldr s2, [a_ptr1]\n"
- "ldr s4, [a_ptr2]\n"
- "ldr s6, [a_ptr3]\n"
- "ldr s8, [a_ptr4]\n"
- "ldr s10, [a_ptr5]\n"
- "ldr s12, [a_ptr6]\n"
- "ldr s14, [a_ptr7]\n"
+ "ldr s1, [a_ptr1]\n"
+ "ldr s2, [a_ptr2]\n"
+ "ldr s3, [a_ptr3]\n"
+ "ldr s4, [a_ptr4]\n"
+ "ldr s5, [a_ptr5]\n"
+ "ldr s6, [a_ptr6]\n"
+ "ldr s7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b0, [%[a_ptr0]]\n"
- "ldr b2, [a_ptr1]\n"
- "ldr b4, [a_ptr2]\n"
- "ldr b6, [a_ptr3]\n"
- "ldr b8, [a_ptr4]\n"
- "ldr b10, [a_ptr5]\n"
- "ldr b12, [a_ptr6]\n"
- "ldr b14, [a_ptr7]\n"
+ "ldr b1, [a_ptr1]\n"
+ "ldr b2, [a_ptr2]\n"
+ "ldr b3, [a_ptr3]\n"
+ "ldr b4, [a_ptr4]\n"
+ "ldr b5, [a_ptr5]\n"
+ "ldr b6, [a_ptr6]\n"
+ "ldr b7, [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ldr h0, [%[a_ptr0]], #0x2\n"
- "ldr h2, [a_ptr1], #0x2\n"
- "ldr h4, [a_ptr2], #0x2\n"
- "ldr h6, [a_ptr3], #0x2\n"
- "ldr h8, [a_ptr4], #0x2\n"
- "ldr h10, [a_ptr5], #0x2\n"
- "ldr h12, [a_ptr6], #0x2\n"
- "ldr h14, [a_ptr7], #0x2\n"
+ "ldr h1, [a_ptr1], #0x2\n"
+ "ldr h2, [a_ptr2], #0x2\n"
+ "ldr h3, [a_ptr3], #0x2\n"
+ "ldr h4, [a_ptr4], #0x2\n"
+ "ldr h5, [a_ptr5], #0x2\n"
+ "ldr h6, [a_ptr6], #0x2\n"
+ "ldr h7, [a_ptr7], #0x2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v2.b}[2], [a_ptr1]\n"
- "ld1 {v4.b}[2], [a_ptr2]\n"
- "ld1 {v6.b}[2], [a_ptr3]\n"
- "ld1 {v8.b}[2], [a_ptr4]\n"
- "ld1 {v10.b}[2], [a_ptr5]\n"
- "ld1 {v12.b}[2], [a_ptr6]\n"
- "ld1 {v14.b}[2], [a_ptr7]\n"
+ "ld1 {v1.b}[2], [a_ptr1]\n"
+ "ld1 {v2.b}[2], [a_ptr2]\n"
+ "ld1 {v3.b}[2], [a_ptr3]\n"
+ "ld1 {v4.b}[2], [a_ptr4]\n"
+ "ld1 {v5.b}[2], [a_ptr5]\n"
+ "ld1 {v6.b}[2], [a_ptr6]\n"
+ "ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -168,14 +169,14 @@
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -190,33 +191,33 @@
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
@@ -233,32 +234,32 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -356,55 +357,55 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr d0, [%[a_ptr0]]\n"
- "ldr d2, [a_ptr1]\n"
- "ldr d4, [a_ptr2]\n"
- "ldr d6, [a_ptr3]\n"
- "ldr d8, [a_ptr4]\n"
- "ldr d10, [a_ptr5]\n"
- "ldr d12, [a_ptr6]\n"
- "ldr d14, [a_ptr7]\n"
+ "ldr d1, [a_ptr1]\n"
+ "ldr d2, [a_ptr2]\n"
+ "ldr d3, [a_ptr3]\n"
+ "ldr d4, [a_ptr4]\n"
+ "ldr d5, [a_ptr5]\n"
+ "ldr d6, [a_ptr6]\n"
+ "ldr d7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr s0, [%[a_ptr0]], #0x4\n"
- "ldr s2, [a_ptr1], #0x4\n"
- "ldr s4, [a_ptr2], #0x4\n"
- "ldr s6, [a_ptr3], #0x4\n"
- "ldr s8, [a_ptr4], #0x4\n"
- "ldr s10, [a_ptr5], #0x4\n"
- "ldr s12, [a_ptr6], #0x4\n"
- "ldr s14, [a_ptr7], #0x4\n"
+ "ldr s1, [a_ptr1], #0x4\n"
+ "ldr s2, [a_ptr2], #0x4\n"
+ "ldr s3, [a_ptr3], #0x4\n"
+ "ldr s4, [a_ptr4], #0x4\n"
+ "ldr s5, [a_ptr5], #0x4\n"
+ "ldr s6, [a_ptr6], #0x4\n"
+ "ldr s7, [a_ptr7], #0x4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[4], [%[a_ptr0]]\n"
- "ld1 {v2.b}[4], [a_ptr1]\n"
- "ld1 {v4.b}[4], [a_ptr2]\n"
- "ld1 {v6.b}[4], [a_ptr3]\n"
- "ld1 {v8.b}[4], [a_ptr4]\n"
- "ld1 {v10.b}[4], [a_ptr5]\n"
- "ld1 {v12.b}[4], [a_ptr6]\n"
- "ld1 {v14.b}[4], [a_ptr7]\n"
+ "ld1 {v1.b}[4], [a_ptr1]\n"
+ "ld1 {v2.b}[4], [a_ptr2]\n"
+ "ld1 {v3.b}[4], [a_ptr3]\n"
+ "ld1 {v4.b}[4], [a_ptr4]\n"
+ "ld1 {v5.b}[4], [a_ptr5]\n"
+ "ld1 {v6.b}[4], [a_ptr6]\n"
+ "ld1 {v7.b}[4], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[2], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[2], [a_ptr1], #2\n"
- "ld1 {v4.h}[2], [a_ptr2], #2\n"
- "ld1 {v6.h}[2], [a_ptr3], #2\n"
- "ld1 {v8.h}[2], [a_ptr4], #2\n"
- "ld1 {v10.h}[2], [a_ptr5], #2\n"
- "ld1 {v12.h}[2], [a_ptr6], #2\n"
- "ld1 {v14.h}[2], [a_ptr7], #2\n"
+ "ld1 {v1.h}[2], [a_ptr1], #2\n"
+ "ld1 {v2.h}[2], [a_ptr2], #2\n"
+ "ld1 {v3.h}[2], [a_ptr3], #2\n"
+ "ld1 {v4.h}[2], [a_ptr4], #2\n"
+ "ld1 {v5.h}[2], [a_ptr5], #2\n"
+ "ld1 {v6.h}[2], [a_ptr6], #2\n"
+ "ld1 {v7.h}[2], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[6], [%[a_ptr0]]\n"
- "ld1 {v2.b}[6], [a_ptr1]\n"
- "ld1 {v4.b}[6], [a_ptr2]\n"
- "ld1 {v6.b}[6], [a_ptr3]\n"
- "ld1 {v8.b}[6], [a_ptr4]\n"
- "ld1 {v10.b}[6], [a_ptr5]\n"
- "ld1 {v12.b}[6], [a_ptr6]\n"
- "ld1 {v14.b}[6], [a_ptr7]\n"
+ "ld1 {v1.b}[6], [a_ptr1]\n"
+ "ld1 {v2.b}[6], [a_ptr2]\n"
+ "ld1 {v3.b}[6], [a_ptr3]\n"
+ "ld1 {v4.b}[6], [a_ptr4]\n"
+ "ld1 {v5.b}[6], [a_ptr5]\n"
+ "ld1 {v6.b}[6], [a_ptr6]\n"
+ "ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -422,23 +423,23 @@
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -453,49 +454,49 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
@@ -506,40 +507,40 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -636,56 +637,56 @@
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"cbnz %[odds], 2f\n"
"ld1 {v0.s}[2], [%[a_ptr0]]\n"
- "ld1 {v2.s}[2], [a_ptr1]\n"
- "ld1 {v4.s}[2], [a_ptr2]\n"
- "ld1 {v6.s}[2], [a_ptr3]\n"
- "ld1 {v8.s}[2], [a_ptr4]\n"
- "ld1 {v10.s}[2], [a_ptr5]\n"
- "ld1 {v12.s}[2], [a_ptr6]\n"
- "ld1 {v14.s}[2], [a_ptr7]\n"
+ "ld1 {v1.s}[2], [a_ptr1]\n"
+ "ld1 {v2.s}[2], [a_ptr2]\n"
+ "ld1 {v3.s}[2], [a_ptr3]\n"
+ "ld1 {v4.s}[2], [a_ptr4]\n"
+ "ld1 {v5.s}[2], [a_ptr5]\n"
+ "ld1 {v6.s}[2], [a_ptr6]\n"
+ "ld1 {v7.s}[2], [a_ptr7]\n"
"b 3f\n"
"2:\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[8], [%[a_ptr0]]\n"
- "ld1 {v2.b}[8], [a_ptr1]\n"
- "ld1 {v4.b}[8], [a_ptr2]\n"
- "ld1 {v6.b}[8], [a_ptr3]\n"
- "ld1 {v8.b}[8], [a_ptr4]\n"
- "ld1 {v10.b}[8], [a_ptr5]\n"
- "ld1 {v12.b}[8], [a_ptr6]\n"
- "ld1 {v14.b}[8], [a_ptr7]\n"
+ "ld1 {v1.b}[8], [a_ptr1]\n"
+ "ld1 {v2.b}[8], [a_ptr2]\n"
+ "ld1 {v3.b}[8], [a_ptr3]\n"
+ "ld1 {v4.b}[8], [a_ptr4]\n"
+ "ld1 {v5.b}[8], [a_ptr5]\n"
+ "ld1 {v6.b}[8], [a_ptr6]\n"
+ "ld1 {v7.b}[8], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[4], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[4], [a_ptr1], #2\n"
- "ld1 {v4.h}[4], [a_ptr2], #2\n"
- "ld1 {v6.h}[4], [a_ptr3], #2\n"
- "ld1 {v8.h}[4], [a_ptr4], #2\n"
- "ld1 {v10.h}[4], [a_ptr5], #2\n"
- "ld1 {v12.h}[4], [a_ptr6], #2\n"
- "ld1 {v14.h}[4], [a_ptr7], #2\n"
+ "ld1 {v1.h}[4], [a_ptr1], #2\n"
+ "ld1 {v2.h}[4], [a_ptr2], #2\n"
+ "ld1 {v3.h}[4], [a_ptr3], #2\n"
+ "ld1 {v4.h}[4], [a_ptr4], #2\n"
+ "ld1 {v5.h}[4], [a_ptr5], #2\n"
+ "ld1 {v6.h}[4], [a_ptr6], #2\n"
+ "ld1 {v7.h}[4], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[10], [%[a_ptr0]]\n"
- "ld1 {v2.b}[10], [a_ptr1]\n"
- "ld1 {v4.b}[10], [a_ptr2]\n"
- "ld1 {v6.b}[10], [a_ptr3]\n"
- "ld1 {v8.b}[10], [a_ptr4]\n"
- "ld1 {v10.b}[10], [a_ptr5]\n"
- "ld1 {v12.b}[10], [a_ptr6]\n"
- "ld1 {v14.b}[10], [a_ptr7]\n"
+ "ld1 {v1.b}[10], [a_ptr1]\n"
+ "ld1 {v2.b}[10], [a_ptr2]\n"
+ "ld1 {v3.b}[10], [a_ptr3]\n"
+ "ld1 {v4.b}[10], [a_ptr4]\n"
+ "ld1 {v5.b}[10], [a_ptr5]\n"
+ "ld1 {v6.b}[10], [a_ptr6]\n"
+ "ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -703,32 +704,32 @@
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -744,58 +745,58 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
@@ -806,48 +807,48 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -945,63 +946,63 @@
"1:\n"
"cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]]\n"
- "ldr q2, [a_ptr1]\n"
- "ldr q4, [a_ptr2]\n"
- "ldr q6, [a_ptr3]\n"
- "ldr q8, [a_ptr4]\n"
- "ldr q10, [a_ptr5]\n"
- "ldr q12, [a_ptr6]\n"
- "ldr q14, [a_ptr7]\n"
+ "ldr q1, [a_ptr1]\n"
+ "ldr q2, [a_ptr2]\n"
+ "ldr q3, [a_ptr3]\n"
+ "ldr q4, [a_ptr4]\n"
+ "ldr q5, [a_ptr5]\n"
+ "ldr q6, [a_ptr6]\n"
+ "ldr q7, [a_ptr7]\n"
"b 3f\n"
"2:\n"
"ldr d0, [%[a_ptr0]], #0x8\n"
- "ldr d2, [a_ptr1], #0x8\n"
- "ldr d4, [a_ptr2], #0x8\n"
- "ldr d6, [a_ptr3], #0x8\n"
- "ldr d8, [a_ptr4], #0x8\n"
- "ldr d10, [a_ptr5], #0x8\n"
- "ldr d12, [a_ptr6], #0x8\n"
- "ldr d14, [a_ptr7], #0x8\n"
+ "ldr d1, [a_ptr1], #0x8\n"
+ "ldr d2, [a_ptr2], #0x8\n"
+ "ldr d3, [a_ptr3], #0x8\n"
+ "ldr d4, [a_ptr4], #0x8\n"
+ "ldr d5, [a_ptr5], #0x8\n"
+ "ldr d6, [a_ptr6], #0x8\n"
+ "ldr d7, [a_ptr7], #0x8\n"
"ld1 {v0.s}[2], [%[a_ptr0]], #4\n"
- "ld1 {v2.s}[2], [a_ptr1], #4\n"
- "ld1 {v4.s}[2], [a_ptr2], #4\n"
- "ld1 {v6.s}[2], [a_ptr3], #4\n"
- "ld1 {v8.s}[2], [a_ptr4], #4\n"
- "ld1 {v10.s}[2], [a_ptr5], #4\n"
- "ld1 {v12.s}[2], [a_ptr6], #4\n"
- "ld1 {v14.s}[2], [a_ptr7], #4\n"
+ "ld1 {v1.s}[2], [a_ptr1], #4\n"
+ "ld1 {v2.s}[2], [a_ptr2], #4\n"
+ "ld1 {v3.s}[2], [a_ptr3], #4\n"
+ "ld1 {v4.s}[2], [a_ptr4], #4\n"
+ "ld1 {v5.s}[2], [a_ptr5], #4\n"
+ "ld1 {v6.s}[2], [a_ptr6], #4\n"
+ "ld1 {v7.s}[2], [a_ptr7], #4\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v0.b}[12], [%[a_ptr0]]\n"
- "ld1 {v2.b}[12], [a_ptr1]\n"
- "ld1 {v4.b}[12], [a_ptr2]\n"
- "ld1 {v6.b}[12], [a_ptr3]\n"
- "ld1 {v8.b}[12], [a_ptr4]\n"
- "ld1 {v10.b}[12], [a_ptr5]\n"
- "ld1 {v12.b}[12], [a_ptr6]\n"
- "ld1 {v14.b}[12], [a_ptr7]\n"
+ "ld1 {v1.b}[12], [a_ptr1]\n"
+ "ld1 {v2.b}[12], [a_ptr2]\n"
+ "ld1 {v3.b}[12], [a_ptr3]\n"
+ "ld1 {v4.b}[12], [a_ptr4]\n"
+ "ld1 {v5.b}[12], [a_ptr5]\n"
+ "ld1 {v6.b}[12], [a_ptr6]\n"
+ "ld1 {v7.b}[12], [a_ptr7]\n"
"b 3f\n"
"4:\n"
"ld1 {v0.h}[6], [%[a_ptr0]], #2\n"
- "ld1 {v2.h}[6], [a_ptr1], #2\n"
- "ld1 {v4.h}[6], [a_ptr2], #2\n"
- "ld1 {v6.h}[6], [a_ptr3], #2\n"
- "ld1 {v8.h}[6], [a_ptr4], #2\n"
- "ld1 {v10.h}[6], [a_ptr5], #2\n"
- "ld1 {v12.h}[6], [a_ptr6], #2\n"
- "ld1 {v14.h}[6], [a_ptr7], #2\n"
+ "ld1 {v1.h}[6], [a_ptr1], #2\n"
+ "ld1 {v2.h}[6], [a_ptr2], #2\n"
+ "ld1 {v3.h}[6], [a_ptr3], #2\n"
+ "ld1 {v4.h}[6], [a_ptr4], #2\n"
+ "ld1 {v5.h}[6], [a_ptr5], #2\n"
+ "ld1 {v6.h}[6], [a_ptr6], #2\n"
+ "ld1 {v7.h}[6], [a_ptr7], #2\n"
"subs %[odds], %[odds], #0x1\n"
"b.ne 5f\n"
"b 3f\n"
"5:\n"
"ld1 {v0.b}[14], [%[a_ptr0]]\n"
- "ld1 {v2.b}[14], [a_ptr1]\n"
- "ld1 {v4.b}[14], [a_ptr2]\n"
- "ld1 {v6.b}[14], [a_ptr3]\n"
- "ld1 {v8.b}[14], [a_ptr4]\n"
- "ld1 {v10.b}[14], [a_ptr5]\n"
- "ld1 {v12.b}[14], [a_ptr6]\n"
- "ld1 {v14.b}[14], [a_ptr7]\n"
+ "ld1 {v1.b}[14], [a_ptr1]\n"
+ "ld1 {v2.b}[14], [a_ptr2]\n"
+ "ld1 {v3.b}[14], [a_ptr3]\n"
+ "ld1 {v4.b}[14], [a_ptr4]\n"
+ "ld1 {v5.b}[14], [a_ptr5]\n"
+ "ld1 {v6.b}[14], [a_ptr6]\n"
+ "ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
"movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
@@ -1019,41 +1020,41 @@
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1070,67 +1071,67 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
@@ -1141,56 +1142,56 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1354,50 +1355,50 @@
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1415,76 +1416,76 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
@@ -1495,64 +1496,64 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -1724,59 +1725,59 @@
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -1795,85 +1796,85 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
@@ -1884,72 +1885,72 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -2121,68 +2122,68 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2202,94 +2203,94 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"ldr q21, [%[b_ptr0], #0x50]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
@@ -2300,80 +2301,80 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
@@ -2554,77 +2555,77 @@
"ldr q22, [%[b_ptr0], #0x60]\n"
"movi v31.4s, #0\n"
"ldr q23, [%[b_ptr0], #0x70]\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
"cbz %[loops], 6f\n"
"ldr q16, [%[b_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
@@ -2645,103 +2646,103 @@
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v25.4s, #0\n"
"add c_ptr1, c_ptr1, #0x10\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
"ldr q16, [%[b_ptr0]]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
"ldr q18, [%[b_ptr0], #0x20]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
"ldr q19, [%[b_ptr0], #0x30]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
"ldr q20, [%[b_ptr0], #0x40]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
"ldr q21, [%[b_ptr0], #0x50]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
"ldr q22, [%[b_ptr0], #0x60]\n"
- ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
@@ -2752,88 +2753,88 @@
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
- ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
- ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
- ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
- ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
- ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
- ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
- ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
- ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
- ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
- ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
- ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
- ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
- ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
- ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
- ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
- ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
- ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
- ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
- ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
- ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
- ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
- ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
- ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
- ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
- ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
- ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
- ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
- ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
- ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
- ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
- ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
- ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
- ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
"6:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp
index c6895a6..28ef807 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,7 +32,7 @@
{
// Actual kernel implementations
-void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int);
+void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool);
class hybrid_fp16_mla_4VLx4
{
@@ -40,10 +40,10 @@
typedef __fp16 operand_type;
typedef __fp16 result_type;
- typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int);
+ typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -53,20 +53,32 @@
return get_vector_length<__fp16>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {};
// Default to the generic kernel
kern_type kernel=sve_hybrid_fp16_mla_4VLx4;
- hybrid_fp16_mla_4VLx4(const CPUInfo *ci)
- {
-
- }
+ hybrid_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp
index ab41fb3..2998f33 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,37 +25,58 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
+void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) {
const int K_stride = K;
const long loops_count = ((K + 8) / 16) - 1;
K -= loops_count * 16;
const long regs_count = (K / 8) - 1;
K -= (regs_count + 1) * 8;
const long leftovers = K;
+ __fp16 nullbias[512];
+ if (!append && !bias) {
+ memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16)));
+ }
+ __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ const __fp16 * const minptr = &minval;
+ const __fp16 * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<__fp16>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
for (int y=0; y<M; y+=4) {
const __fp16 * const a_ptr0_base = A + (y * lda);
const unsigned long ldab = lda * sizeof(__fp16);
__fp16 *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(__fp16);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<__fp16>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>()));
- const __fp16 *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
long blocks = leftovers;
const __fp16 *a_ptr0 = a_ptr0_base;
const __fp16 *b_ptr0 = B + (K_stride * x0);
+ const unsigned long ldcb = ldc * sizeof(__fp16);
+ const __fp16 *biasptr = bias ? bias+x0 : nullbias;
switch(M-y) {
case 1:
@@ -69,44 +90,37 @@
"whilelt p2.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z18.h, #0\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z19.h, #0\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
"ld1h z16.h, p0/z, [%[c_ptr0]]\n"
"ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -322,13 +336,12 @@
"fmla z19.h, z11.h, z4.h[0]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[1]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
+ "fmla z17.h, z13.h, z4.h[1]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
+ "fmla z18.h, z14.h, z4.h[1]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.h, z15.h, z4.h[1]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[2]\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
@@ -339,9 +352,11 @@
"fmla z19.h, z11.h, z4.h[2]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
"fmla z17.h, z13.h, z4.h[3]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"fmla z18.h, z14.h, z4.h[3]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z19.h, z15.h, z4.h[3]\n"
"ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
@@ -372,50 +387,30 @@
"fmla z19.h, z11.h, z4.h[6]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[7]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z17.h, z13.h, z4.h[7]\n"
"fmla z18.h, z14.h, z4.h[7]\n"
"fmla z19.h, z15.h, z4.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "b.eq 5f\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
+ "fmla z16.h, z8.h, z0.h[0]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
+ "fmla z18.h, z10.h, z0.h[0]\n"
+ "fmla z19.h, z11.h, z0.h[0]\n"
"b.eq 5f\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
+ "fmla z16.h, z12.h, z0.h[1]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
+ "fmla z17.h, z13.h, z0.h[1]\n"
+ "fmla z18.h, z14.h, z0.h[1]\n"
+ "fmla z19.h, z15.h, z0.h[1]\n"
"b.eq 5f\n"
"addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
@@ -423,25 +418,46 @@
"ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
+ "fmla z16.h, z8.h, z0.h[2]\n"
+ "fmla z17.h, z9.h, z0.h[2]\n"
+ "fmla z18.h, z10.h, z0.h[2]\n"
+ "fmla z19.h, z11.h, z0.h[2]\n"
"b.eq 5f\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
+ "fmla z16.h, z12.h, z0.h[3]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z17.h, z13.h, z0.h[3]\n"
+ "fmla z18.h, z14.h, z0.h[3]\n"
+ "fmla z19.h, z15.h, z0.h[3]\n"
+ "b.eq 5f\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.h, z8.h, z0.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.h, z9.h, z0.h[4]\n"
+ "fmla z18.h, z10.h, z0.h[4]\n"
+ "fmla z19.h, z11.h, z0.h[4]\n"
+ "b.eq 5f\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z16.h, z12.h, z0.h[5]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z17.h, z13.h, z0.h[5]\n"
"fmla z18.h, z14.h, z0.h[5]\n"
"fmla z19.h, z15.h, z0.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
"fmla z17.h, z9.h, z0.h[6]\n"
"fmla z18.h, z10.h, z0.h[6]\n"
@@ -451,33 +467,34 @@
"fmla z16.h, z8.h, z0.h[0]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
+ "fmla z18.h, z10.h, z0.h[0]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
+ "fmla z19.h, z11.h, z0.h[0]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
+ "fmla z16.h, z12.h, z0.h[1]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
+ "fmla z17.h, z13.h, z0.h[1]\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
+ "fmla z18.h, z14.h, z0.h[1]\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
+ "fmla z19.h, z15.h, z0.h[1]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
+ "fmla z16.h, z8.h, z0.h[2]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z17.h, z9.h, z0.h[2]\n"
+ "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
"fmla z18.h, z10.h, z0.h[2]\n"
"addvl %[b_ptr0], %[b_ptr0], #16\n"
"fmla z19.h, z11.h, z0.h[2]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
"ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
+ "fmla z17.h, z13.h, z0.h[3]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
+ "fmla z18.h, z14.h, z0.h[3]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.h, z15.h, z0.h[3]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[4]\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
@@ -504,50 +521,30 @@
"fmla z19.h, z11.h, z0.h[6]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[7]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z17.h, z13.h, z0.h[7]\n"
"fmla z18.h, z14.h, z0.h[7]\n"
"fmla z19.h, z15.h, z0.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "b.eq 5f\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
+ "fmla z16.h, z8.h, z4.h[0]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
+ "fmla z17.h, z9.h, z4.h[0]\n"
+ "fmla z18.h, z10.h, z4.h[0]\n"
+ "fmla z19.h, z11.h, z4.h[0]\n"
"b.eq 5f\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
+ "fmla z16.h, z12.h, z4.h[1]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
+ "fmla z17.h, z13.h, z4.h[1]\n"
+ "fmla z18.h, z14.h, z4.h[1]\n"
+ "fmla z19.h, z15.h, z4.h[1]\n"
"b.eq 5f\n"
"addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
@@ -555,37 +552,68 @@
"ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
+ "fmla z16.h, z8.h, z4.h[2]\n"
+ "fmla z17.h, z9.h, z4.h[2]\n"
+ "fmla z18.h, z10.h, z4.h[2]\n"
+ "fmla z19.h, z11.h, z4.h[2]\n"
"b.eq 5f\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
+ "fmla z16.h, z12.h, z4.h[3]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z17.h, z13.h, z4.h[3]\n"
+ "fmla z18.h, z14.h, z4.h[3]\n"
+ "fmla z19.h, z15.h, z4.h[3]\n"
+ "b.eq 5f\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.h, z8.h, z4.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.h, z9.h, z4.h[4]\n"
+ "fmla z18.h, z10.h, z4.h[4]\n"
+ "fmla z19.h, z11.h, z4.h[4]\n"
+ "b.eq 5f\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z16.h, z12.h, z4.h[5]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z17.h, z13.h, z4.h[5]\n"
"fmla z18.h, z14.h, z4.h[5]\n"
"fmla z19.h, z15.h, z4.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[6]\n"
"fmla z17.h, z9.h, z4.h[6]\n"
"fmla z18.h, z10.h, z4.h[6]\n"
"fmla z19.h, z11.h, z4.h[6]\n"
"5:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -604,60 +632,49 @@
"whilelt p2.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "mov z20.d, z16.d\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
+ "mov z21.d, z17.d\n"
"ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z18.h, #0\n"
+ "mov z22.d, z18.d\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z19.h, #0\n"
+ "mov z23.d, z19.d\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.h, #0\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z21.h, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.h, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z23.h, #0\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
"ld1h z16.h, p0/z, [%[c_ptr0]]\n"
"ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
"ld1h z20.h, p0/z, [c_ptr1]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
"ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
"ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
"ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z20.h, p7/m, z20.h, z15.h\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z21.h, p7/m, z21.h, z15.h\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "fmul z22.h, p7/m, z22.h, z15.h\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmul z23.h, p7/m, z23.h, z15.h\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -974,9 +991,11 @@
"fmla z20.h, z8.h, z5.h[0]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z21.h, z9.h, z5.h[0]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z4.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z22.h, z10.h, z5.h[0]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z4.h[0]\n"
@@ -1044,6 +1063,7 @@
"fmla z23.h, z15.h, z5.h[5]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[6]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.h, z8.h, z5.h[6]\n"
"fmla z17.h, z9.h, z4.h[6]\n"
"fmla z21.h, z9.h, z5.h[6]\n"
@@ -1060,13 +1080,12 @@
"fmla z19.h, z15.h, z4.h[7]\n"
"fmla z23.h, z15.h, z5.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[0]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[0]\n"
"fmla z17.h, z9.h, z0.h[0]\n"
"fmla z21.h, z9.h, z1.h[0]\n"
@@ -1075,12 +1094,12 @@
"fmla z19.h, z11.h, z0.h[0]\n"
"fmla z23.h, z11.h, z1.h[0]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[1]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
"fmla z21.h, z13.h, z1.h[1]\n"
@@ -1089,12 +1108,13 @@
"fmla z19.h, z15.h, z0.h[1]\n"
"fmla z23.h, z15.h, z1.h[1]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[2]\n"
"fmla z17.h, z9.h, z0.h[2]\n"
"fmla z21.h, z9.h, z1.h[2]\n"
@@ -1103,12 +1123,12 @@
"fmla z19.h, z11.h, z0.h[2]\n"
"fmla z23.h, z11.h, z1.h[2]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[3]\n"
"fmla z17.h, z13.h, z0.h[3]\n"
"fmla z21.h, z13.h, z1.h[3]\n"
@@ -1117,13 +1137,12 @@
"fmla z19.h, z15.h, z0.h[3]\n"
"fmla z23.h, z15.h, z1.h[3]\n"
"b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[4]\n"
"fmla z17.h, z9.h, z0.h[4]\n"
"fmla z21.h, z9.h, z1.h[4]\n"
@@ -1132,12 +1151,12 @@
"fmla z19.h, z11.h, z0.h[4]\n"
"fmla z23.h, z11.h, z1.h[4]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[5]\n"
"fmla z17.h, z13.h, z0.h[5]\n"
"fmla z21.h, z13.h, z1.h[5]\n"
@@ -1146,10 +1165,11 @@
"fmla z19.h, z15.h, z0.h[5]\n"
"fmla z23.h, z15.h, z1.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
"fmla z20.h, z8.h, z1.h[6]\n"
"fmla z17.h, z9.h, z0.h[6]\n"
@@ -1163,19 +1183,21 @@
"fmla z16.h, z8.h, z0.h[0]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
+ "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.h, z10.h, z0.h[0]\n"
+ "ld1rqh z5.h, p6/z, [a_ptr1]\n"
"fmla z22.h, z10.h, z1.h[0]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z23.h, z11.h, z1.h[0]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z20.h, z12.h, z1.h[1]\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
@@ -1237,6 +1259,7 @@
"fmla z23.h, z15.h, z1.h[5]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.h, z8.h, z1.h[6]\n"
"fmla z17.h, z9.h, z0.h[6]\n"
"fmla z21.h, z9.h, z1.h[6]\n"
@@ -1253,13 +1276,12 @@
"fmla z19.h, z15.h, z0.h[7]\n"
"fmla z23.h, z15.h, z1.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[0]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[0]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
"fmla z21.h, z9.h, z5.h[0]\n"
@@ -1268,12 +1290,12 @@
"fmla z19.h, z11.h, z4.h[0]\n"
"fmla z23.h, z11.h, z5.h[0]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[1]\n"
"fmla z17.h, z13.h, z4.h[1]\n"
"fmla z21.h, z13.h, z5.h[1]\n"
@@ -1282,12 +1304,13 @@
"fmla z19.h, z15.h, z4.h[1]\n"
"fmla z23.h, z15.h, z5.h[1]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[2]\n"
"fmla z17.h, z9.h, z4.h[2]\n"
"fmla z21.h, z9.h, z5.h[2]\n"
@@ -1296,12 +1319,12 @@
"fmla z19.h, z11.h, z4.h[2]\n"
"fmla z23.h, z11.h, z5.h[2]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[3]\n"
"fmla z17.h, z13.h, z4.h[3]\n"
"fmla z21.h, z13.h, z5.h[3]\n"
@@ -1310,13 +1333,12 @@
"fmla z19.h, z15.h, z4.h[3]\n"
"fmla z23.h, z15.h, z5.h[3]\n"
"b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[4]\n"
"fmla z17.h, z9.h, z4.h[4]\n"
"fmla z21.h, z9.h, z5.h[4]\n"
@@ -1325,12 +1347,12 @@
"fmla z19.h, z11.h, z4.h[4]\n"
"fmla z23.h, z11.h, z5.h[4]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[5]\n"
"fmla z17.h, z13.h, z4.h[5]\n"
"fmla z21.h, z13.h, z5.h[5]\n"
@@ -1339,10 +1361,11 @@
"fmla z19.h, z15.h, z4.h[5]\n"
"fmla z23.h, z15.h, z5.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[6]\n"
"fmla z20.h, z8.h, z5.h[6]\n"
"fmla z17.h, z9.h, z4.h[6]\n"
@@ -1352,9 +1375,27 @@
"fmla z19.h, z11.h, z4.h[6]\n"
"fmla z23.h, z11.h, z5.h[6]\n"
"5:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
+ "fmax z20.h, p7/m, z20.h, z14.h\n"
+ "fmax z21.h, p7/m, z21.h, z14.h\n"
+ "fmax z22.h, p7/m, z22.h, z14.h\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.h, p7/m, z23.h, z14.h\n"
+ "fmin z20.h, p7/m, z20.h, z15.h\n"
+ "fmin z21.h, p7/m, z21.h, z15.h\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.h, p7/m, z22.h, z15.h\n"
+ "fmin z23.h, p7/m, z23.h, z15.h\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
"st1h z20.h, p0, [c_ptr1]\n"
@@ -1364,7 +1405,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -1387,76 +1428,61 @@
"whilelt p2.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "mov z20.d, z16.d\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
+ "mov z21.d, z17.d\n"
"ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z18.h, #0\n"
+ "mov z22.d, z18.d\n"
"ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z19.h, #0\n"
+ "mov z23.d, z19.d\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z20.h, #0\n"
+ "mov z24.d, z16.d\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.h, #0\n"
+ "mov z25.d, z17.d\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z22.h, #0\n"
+ "mov z26.d, z18.d\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z23.h, #0\n"
+ "mov z27.d, z19.d\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z24.h, #0\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z25.h, #0\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z26.h, #0\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "mov z27.h, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"add a_ptr1, a_ptr1, #0x10\n"
"add a_ptr2, a_ptr2, #0x10\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
"ld1h z16.h, p0/z, [%[c_ptr0]]\n"
"ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
"ld1h z20.h, p0/z, [c_ptr1]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
"ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
"ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
"ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z20.h, p7/m, z20.h, z15.h\n"
"ld1h z24.h, p0/z, [c_ptr2]\n"
- "fmul z21.h, p7/m, z21.h, z15.h\n"
"ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z22.h, p7/m, z22.h, z15.h\n"
"ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z23.h, p7/m, z23.h, z15.h\n"
"ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z24.h, p7/m, z24.h, z15.h\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z25.h, p7/m, z25.h, z15.h\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "fmul z26.h, p7/m, z26.h, z15.h\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "fmul z27.h, p7/m, z27.h, z15.h\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -1872,10 +1898,13 @@
"fmla z16.h, z8.h, z4.h[0]\n"
"ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
"fmla z20.h, z8.h, z5.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z24.h, z8.h, z6.h[0]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z21.h, z9.h, z5.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z25.h, z9.h, z6.h[0]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z4.h[0]\n"
@@ -1968,6 +1997,7 @@
"fmla z27.h, z15.h, z6.h[5]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[6]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.h, z8.h, z5.h[6]\n"
"fmla z24.h, z8.h, z6.h[6]\n"
"fmla z17.h, z9.h, z4.h[6]\n"
@@ -1992,13 +2022,12 @@
"fmla z23.h, z15.h, z5.h[7]\n"
"fmla z27.h, z15.h, z6.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[0]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[0]\n"
"fmla z24.h, z8.h, z2.h[0]\n"
"fmla z17.h, z9.h, z0.h[0]\n"
@@ -2011,12 +2040,12 @@
"fmla z23.h, z11.h, z1.h[0]\n"
"fmla z27.h, z11.h, z2.h[0]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[1]\n"
"fmla z24.h, z12.h, z2.h[1]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
@@ -2029,12 +2058,13 @@
"fmla z23.h, z15.h, z1.h[1]\n"
"fmla z27.h, z15.h, z2.h[1]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[2]\n"
"fmla z24.h, z8.h, z2.h[2]\n"
"fmla z17.h, z9.h, z0.h[2]\n"
@@ -2047,12 +2077,12 @@
"fmla z23.h, z11.h, z1.h[2]\n"
"fmla z27.h, z11.h, z2.h[2]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[3]\n"
"fmla z24.h, z12.h, z2.h[3]\n"
"fmla z17.h, z13.h, z0.h[3]\n"
@@ -2065,13 +2095,12 @@
"fmla z23.h, z15.h, z1.h[3]\n"
"fmla z27.h, z15.h, z2.h[3]\n"
"b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[4]\n"
"fmla z24.h, z8.h, z2.h[4]\n"
"fmla z17.h, z9.h, z0.h[4]\n"
@@ -2084,12 +2113,12 @@
"fmla z23.h, z11.h, z1.h[4]\n"
"fmla z27.h, z11.h, z2.h[4]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[5]\n"
"fmla z24.h, z12.h, z2.h[5]\n"
"fmla z17.h, z13.h, z0.h[5]\n"
@@ -2102,10 +2131,11 @@
"fmla z23.h, z15.h, z1.h[5]\n"
"fmla z27.h, z15.h, z2.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
"fmla z20.h, z8.h, z1.h[6]\n"
"fmla z24.h, z8.h, z2.h[6]\n"
@@ -2125,18 +2155,21 @@
"fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
"fmla z24.h, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
+ "ld1rqh z5.h, p6/z, [a_ptr1]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
+ "ld1rqh z6.h, p6/z, [a_ptr2]\n"
"fmla z25.h, z9.h, z2.h[0]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z22.h, z10.h, z1.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z26.h, z10.h, z2.h[0]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"fmla z23.h, z11.h, z1.h[0]\n"
"fmla z27.h, z11.h, z2.h[0]\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
@@ -2222,6 +2255,7 @@
"fmla z27.h, z15.h, z2.h[5]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.h, z8.h, z1.h[6]\n"
"fmla z24.h, z8.h, z2.h[6]\n"
"fmla z17.h, z9.h, z0.h[6]\n"
@@ -2246,13 +2280,12 @@
"fmla z23.h, z15.h, z1.h[7]\n"
"fmla z27.h, z15.h, z2.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[0]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[0]\n"
"fmla z24.h, z8.h, z6.h[0]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
@@ -2265,12 +2298,12 @@
"fmla z23.h, z11.h, z5.h[0]\n"
"fmla z27.h, z11.h, z6.h[0]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[1]\n"
"fmla z24.h, z12.h, z6.h[1]\n"
"fmla z17.h, z13.h, z4.h[1]\n"
@@ -2283,12 +2316,13 @@
"fmla z23.h, z15.h, z5.h[1]\n"
"fmla z27.h, z15.h, z6.h[1]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[2]\n"
"fmla z24.h, z8.h, z6.h[2]\n"
"fmla z17.h, z9.h, z4.h[2]\n"
@@ -2301,12 +2335,12 @@
"fmla z23.h, z11.h, z5.h[2]\n"
"fmla z27.h, z11.h, z6.h[2]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[3]\n"
"fmla z24.h, z12.h, z6.h[3]\n"
"fmla z17.h, z13.h, z4.h[3]\n"
@@ -2319,13 +2353,12 @@
"fmla z23.h, z15.h, z5.h[3]\n"
"fmla z27.h, z15.h, z6.h[3]\n"
"b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[4]\n"
"fmla z24.h, z8.h, z6.h[4]\n"
"fmla z17.h, z9.h, z4.h[4]\n"
@@ -2338,12 +2371,12 @@
"fmla z23.h, z11.h, z5.h[4]\n"
"fmla z27.h, z11.h, z6.h[4]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[5]\n"
"fmla z24.h, z12.h, z6.h[5]\n"
"fmla z17.h, z13.h, z4.h[5]\n"
@@ -2356,10 +2389,11 @@
"fmla z23.h, z15.h, z5.h[5]\n"
"fmla z27.h, z15.h, z6.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[6]\n"
"fmla z20.h, z8.h, z5.h[6]\n"
"fmla z24.h, z8.h, z6.h[6]\n"
@@ -2373,13 +2407,39 @@
"fmla z23.h, z11.h, z5.h[6]\n"
"fmla z27.h, z11.h, z6.h[6]\n"
"5:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
+ "fmax z20.h, p7/m, z20.h, z14.h\n"
+ "fmax z21.h, p7/m, z21.h, z14.h\n"
+ "fmax z22.h, p7/m, z22.h, z14.h\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.h, p7/m, z23.h, z14.h\n"
+ "fmin z20.h, p7/m, z20.h, z15.h\n"
+ "fmin z21.h, p7/m, z21.h, z15.h\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.h, p7/m, z22.h, z15.h\n"
+ "fmin z23.h, p7/m, z23.h, z15.h\n"
+ "fmax z24.h, p7/m, z24.h, z14.h\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.h, p7/m, z25.h, z14.h\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.h, p7/m, z26.h, z14.h\n"
"st1h z20.h, p0, [c_ptr1]\n"
+ "fmin z24.h, p7/m, z24.h, z15.h\n"
+ "fmin z25.h, p7/m, z25.h, z15.h\n"
+ "fmax z27.h, p7/m, z27.h, z14.h\n"
"st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.h, p7/m, z26.h, z15.h\n"
+ "fmin z27.h, p7/m, z27.h, z15.h\n"
"st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n"
"st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n"
"st1h z24.h, p0, [c_ptr2]\n"
@@ -2391,7 +2451,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -2419,92 +2479,73 @@
"whilelt p2.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "mov z20.d, z16.d\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
+ "mov z21.d, z17.d\n"
"ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z18.h, #0\n"
+ "mov z22.d, z18.d\n"
"ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z19.h, #0\n"
+ "mov z23.d, z19.d\n"
"ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "mov z20.h, #0\n"
+ "mov z24.d, z16.d\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z21.h, #0\n"
+ "mov z25.d, z17.d\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z22.h, #0\n"
+ "mov z26.d, z18.d\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z23.h, #0\n"
+ "mov z27.d, z19.d\n"
"ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z24.h, #0\n"
+ "mov z28.d, z16.d\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z25.h, #0\n"
+ "mov z29.d, z17.d\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z26.h, #0\n"
+ "mov z30.d, z18.d\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z27.h, #0\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "mov z28.h, #0\n"
+ "mov z31.d, z19.d\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z29.h, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "mov z30.h, #0\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "mov z31.h, #0\n"
"add a_ptr3, a_ptr3, #0x10\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
"ld1h z16.h, p0/z, [%[c_ptr0]]\n"
"ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
"ld1h z20.h, p0/z, [c_ptr1]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
"ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
"ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
"ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z20.h, p7/m, z20.h, z15.h\n"
"ld1h z24.h, p0/z, [c_ptr2]\n"
- "fmul z21.h, p7/m, z21.h, z15.h\n"
"ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z22.h, p7/m, z22.h, z15.h\n"
"ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z23.h, p7/m, z23.h, z15.h\n"
"ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z24.h, p7/m, z24.h, z15.h\n"
"ld1h z28.h, p0/z, [c_ptr3]\n"
- "fmul z25.h, p7/m, z25.h, z15.h\n"
"ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n"
- "fmul z26.h, p7/m, z26.h, z15.h\n"
"ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n"
- "fmul z27.h, p7/m, z27.h, z15.h\n"
"ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n"
- "fmul z28.h, p7/m, z28.h, z15.h\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z29.h, p7/m, z29.h, z15.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "fmul z30.h, p7/m, z30.h, z15.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "fmul z31.h, p7/m, z31.h, z15.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1rqh z1.h, p7/z, [a_ptr1]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1rqh z2.h, p7/z, [a_ptr2]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1rqh z3.h, p7/z, [a_ptr3]\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -3021,11 +3062,15 @@
"fmla z16.h, z8.h, z4.h[0]\n"
"ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
"fmla z20.h, z8.h, z5.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z24.h, z8.h, z6.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z28.h, z8.h, z7.h[0]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z21.h, z9.h, z5.h[0]\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"fmla z25.h, z9.h, z6.h[0]\n"
"fmla z29.h, z9.h, z7.h[0]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -3141,6 +3186,7 @@
"fmla z31.h, z15.h, z7.h[5]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[6]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.h, z8.h, z5.h[6]\n"
"fmla z24.h, z8.h, z6.h[6]\n"
"fmla z28.h, z8.h, z7.h[6]\n"
@@ -3173,13 +3219,12 @@
"fmla z27.h, z15.h, z6.h[7]\n"
"fmla z31.h, z15.h, z7.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[0]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[0]\n"
"fmla z24.h, z8.h, z2.h[0]\n"
"fmla z28.h, z8.h, z3.h[0]\n"
@@ -3196,12 +3241,12 @@
"fmla z27.h, z11.h, z2.h[0]\n"
"fmla z31.h, z11.h, z3.h[0]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[1]\n"
"fmla z24.h, z12.h, z2.h[1]\n"
"fmla z28.h, z12.h, z3.h[1]\n"
@@ -3218,12 +3263,13 @@
"fmla z27.h, z15.h, z2.h[1]\n"
"fmla z31.h, z15.h, z3.h[1]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[2]\n"
"fmla z24.h, z8.h, z2.h[2]\n"
"fmla z28.h, z8.h, z3.h[2]\n"
@@ -3240,12 +3286,12 @@
"fmla z27.h, z11.h, z2.h[2]\n"
"fmla z31.h, z11.h, z3.h[2]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[3]\n"
"fmla z24.h, z12.h, z2.h[3]\n"
"fmla z28.h, z12.h, z3.h[3]\n"
@@ -3262,13 +3308,12 @@
"fmla z27.h, z15.h, z2.h[3]\n"
"fmla z31.h, z15.h, z3.h[3]\n"
"b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[4]\n"
"fmla z24.h, z8.h, z2.h[4]\n"
"fmla z28.h, z8.h, z3.h[4]\n"
@@ -3285,12 +3330,12 @@
"fmla z27.h, z11.h, z2.h[4]\n"
"fmla z31.h, z11.h, z3.h[4]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z1.h[5]\n"
"fmla z24.h, z12.h, z2.h[5]\n"
"fmla z28.h, z12.h, z3.h[5]\n"
@@ -3307,10 +3352,11 @@
"fmla z27.h, z15.h, z2.h[5]\n"
"fmla z31.h, z15.h, z3.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
"fmla z20.h, z8.h, z1.h[6]\n"
"fmla z24.h, z8.h, z2.h[6]\n"
@@ -3336,17 +3382,21 @@
"fmla z24.h, z8.h, z2.h[0]\n"
"ld1rqh z5.h, p6/z, [a_ptr1]\n"
"fmla z28.h, z8.h, z3.h[0]\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
+ "ld1rqh z6.h, p6/z, [a_ptr2]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
+ "ld1rqh z7.h, p6/z, [a_ptr3]\n"
"fmla z25.h, z9.h, z2.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z29.h, z9.h, z3.h[0]\n"
"ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z22.h, z10.h, z1.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"fmla z26.h, z10.h, z2.h[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"fmla z30.h, z10.h, z3.h[0]\n"
"ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
@@ -3456,6 +3506,7 @@
"fmla z31.h, z15.h, z3.h[5]\n"
"ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.h, z8.h, z1.h[6]\n"
"fmla z24.h, z8.h, z2.h[6]\n"
"fmla z28.h, z8.h, z3.h[6]\n"
@@ -3488,13 +3539,12 @@
"fmla z27.h, z15.h, z2.h[7]\n"
"fmla z31.h, z15.h, z3.h[7]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[0]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[0]\n"
"fmla z24.h, z8.h, z6.h[0]\n"
"fmla z28.h, z8.h, z7.h[0]\n"
@@ -3511,12 +3561,12 @@
"fmla z27.h, z11.h, z6.h[0]\n"
"fmla z31.h, z11.h, z7.h[0]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[1]\n"
"fmla z24.h, z12.h, z6.h[1]\n"
"fmla z28.h, z12.h, z7.h[1]\n"
@@ -3533,12 +3583,13 @@
"fmla z27.h, z15.h, z6.h[1]\n"
"fmla z31.h, z15.h, z7.h[1]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[2]\n"
"fmla z24.h, z8.h, z6.h[2]\n"
"fmla z28.h, z8.h, z7.h[2]\n"
@@ -3555,12 +3606,12 @@
"fmla z27.h, z11.h, z6.h[2]\n"
"fmla z31.h, z11.h, z7.h[2]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[3]\n"
"fmla z24.h, z12.h, z6.h[3]\n"
"fmla z28.h, z12.h, z7.h[3]\n"
@@ -3577,13 +3628,12 @@
"fmla z27.h, z15.h, z6.h[3]\n"
"fmla z31.h, z15.h, z7.h[3]\n"
"b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[4]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z5.h[4]\n"
"fmla z24.h, z8.h, z6.h[4]\n"
"fmla z28.h, z8.h, z7.h[4]\n"
@@ -3600,12 +3650,12 @@
"fmla z27.h, z11.h, z6.h[4]\n"
"fmla z31.h, z11.h, z7.h[4]\n"
"b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.h, z12.h, z5.h[5]\n"
"fmla z24.h, z12.h, z6.h[5]\n"
"fmla z28.h, z12.h, z7.h[5]\n"
@@ -3622,10 +3672,11 @@
"fmla z27.h, z15.h, z6.h[5]\n"
"fmla z31.h, z15.h, z7.h[5]\n"
"b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.h, z8.h, z4.h[6]\n"
"fmla z20.h, z8.h, z5.h[6]\n"
"fmla z24.h, z8.h, z6.h[6]\n"
@@ -3643,16 +3694,50 @@
"fmla z27.h, z11.h, z6.h[6]\n"
"fmla z31.h, z11.h, z7.h[6]\n"
"5:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
+ "fmax z20.h, p7/m, z20.h, z14.h\n"
+ "fmax z21.h, p7/m, z21.h, z14.h\n"
+ "fmax z22.h, p7/m, z22.h, z14.h\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.h, p7/m, z23.h, z14.h\n"
+ "fmin z20.h, p7/m, z20.h, z15.h\n"
+ "fmin z21.h, p7/m, z21.h, z15.h\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.h, p7/m, z22.h, z15.h\n"
+ "fmin z23.h, p7/m, z23.h, z15.h\n"
+ "fmax z24.h, p7/m, z24.h, z14.h\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.h, p7/m, z25.h, z14.h\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.h, p7/m, z26.h, z14.h\n"
"st1h z20.h, p0, [c_ptr1]\n"
+ "fmin z24.h, p7/m, z24.h, z15.h\n"
+ "fmin z25.h, p7/m, z25.h, z15.h\n"
+ "fmax z27.h, p7/m, z27.h, z14.h\n"
"st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.h, p7/m, z26.h, z15.h\n"
+ "fmax z28.h, p7/m, z28.h, z14.h\n"
+ "fmax z29.h, p7/m, z29.h, z14.h\n"
"st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n"
+ "fmin z27.h, p7/m, z27.h, z15.h\n"
+ "fmax z30.h, p7/m, z30.h, z14.h\n"
+ "fmin z28.h, p7/m, z28.h, z15.h\n"
"st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n"
+ "fmin z29.h, p7/m, z29.h, z15.h\n"
+ "fmax z31.h, p7/m, z31.h, z14.h\n"
+ "fmin z30.h, p7/m, z30.h, z15.h\n"
"st1h z24.h, p0, [c_ptr2]\n"
+ "fmin z31.h, p7/m, z31.h, z15.h\n"
"st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n"
"st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n"
"st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n"
@@ -3667,11 +3752,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp
index 76f452d..8e3c179 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,7 +32,7 @@
{
// Actual kernel implementations
-void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, float, int, int, int);
+void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
class hybrid_fp32_mla_4VLx4
{
@@ -40,10 +40,10 @@
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int);
+ typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -53,20 +53,32 @@
return get_vector_length<float>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {};
// Default to the generic kernel
kern_type kernel=sve_hybrid_fp32_mla_4VLx4;
- hybrid_fp32_mla_4VLx4(const CPUInfo *ci)
- {
-
- }
+ hybrid_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp
index 1b2d785..855d27a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,37 +25,58 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
+void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) {
const int K_stride = K;
const long loops_count = ((K + 4) / 8) - 1;
K -= loops_count * 8;
const long regs_count = (K / 4) - 1;
K -= (regs_count + 1) * 4;
const long leftovers = K;
+ float nullbias[256];
+ if (!append && !bias) {
+ memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float)));
+ }
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ const float * const minptr = &minval;
+ const float * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
for (int y=0; y<M; y+=4) {
const float * const a_ptr0_base = A + (y * lda);
const unsigned long ldab = lda * sizeof(float);
float *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(float);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>()));
- const float *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
long blocks = leftovers;
const float *a_ptr0 = a_ptr0_base;
const float *b_ptr0 = B + (K_stride * x0);
+ const unsigned long ldcb = ldc * sizeof(float);
+ const float *biasptr = bias ? bias+x0 : nullbias;
switch(M-y) {
case 1:
@@ -69,269 +90,283 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "mov z18.s, #0\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z19.s, #0\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "b 2f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "2:\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 3f\n"
+ "cbz %[loops], 2f\n"
+ "3:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "b.ne 3b\n"
+ "2:\n"
+ "cbz %[regs], 4f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "b 5f\n"
"4:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 4b\n"
- "3:\n"
- "cbz %[regs], 5f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
"fmla z19.s, z15.s, z0.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "b 6f\n"
- "5:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
"ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z19.s, z11.s, z0.s[2]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z16.s, z12.s, z0.s[3]\n"
"fmla z17.s, z13.s, z0.s[3]\n"
"fmla z18.s, z14.s, z0.s[3]\n"
"fmla z19.s, z15.s, z0.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
"fmla z18.s, z10.s, z4.s[2]\n"
"fmla z19.s, z11.s, z4.s[2]\n"
- "6:\n"
+ "5:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -350,83 +385,74 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "mov z20.d, z16.d\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
+ "mov z21.d, z17.d\n"
"ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
+ "mov z22.d, z18.d\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "mov z19.s, #0\n"
+ "mov z23.d, z19.d\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.s, #0\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z21.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z23.s, #0\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "b 2f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "fmul z20.s, p7/m, z20.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "fmul z21.s, p7/m, z21.s, z15.s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmul z22.s, p7/m, z22.s, z15.s\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmul z23.s, p7/m, z23.s, z15.s\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "2:\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 3f\n"
- "4:\n"
+ "cbz %[loops], 2f\n"
+ "3:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z22.s, z10.s, z1.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
+ "subs %[loops], %[loops], #0x1\n"
"fmla z23.s, z11.s, z1.s[0]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z20.s, z12.s, z1.s[1]\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"fmla z21.s, z13.s, z1.s[1]\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
@@ -440,12 +466,12 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
"fmla z23.s, z11.s, z1.s[2]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[3]\n"
@@ -491,12 +517,12 @@
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
"fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
"fmla z23.s, z11.s, z5.s[2]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z12.s, z4.s[3]\n"
@@ -510,19 +536,19 @@
"ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 4b\n"
- "3:\n"
- "cbz %[regs], 5f\n"
+ "b.ne 3b\n"
+ "2:\n"
+ "cbz %[regs], 4f\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z22.s, z10.s, z1.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
@@ -545,12 +571,12 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
"fmla z23.s, z11.s, z1.s[2]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[3]\n"
@@ -571,9 +597,11 @@
"fmla z20.s, z8.s, z5.s[0]\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z21.s, z9.s, z5.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z4.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z22.s, z10.s, z5.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z4.s[0]\n"
@@ -592,6 +620,7 @@
"fmla z23.s, z15.s, z5.s[1]\n"
"ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
"fmla z21.s, z9.s, z5.s[2]\n"
@@ -607,14 +636,13 @@
"fmla z22.s, z14.s, z5.s[3]\n"
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
@@ -622,13 +650,13 @@
"fmla z22.s, z10.s, z1.s[0]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
"fmla z23.s, z11.s, z1.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
"fmla z21.s, z13.s, z1.s[1]\n"
@@ -636,11 +664,12 @@
"fmla z22.s, z14.s, z1.s[1]\n"
"fmla z19.s, z15.s, z0.s[1]\n"
"fmla z23.s, z15.s, z1.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
@@ -649,23 +678,26 @@
"fmla z22.s, z10.s, z1.s[2]\n"
"fmla z19.s, z11.s, z0.s[2]\n"
"fmla z23.s, z11.s, z1.s[2]\n"
- "b 6f\n"
- "5:\n"
+ "b 5f\n"
+ "4:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
"fmla z22.s, z10.s, z1.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z23.s, z11.s, z1.s[0]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z20.s, z12.s, z1.s[1]\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
@@ -678,6 +710,7 @@
"fmla z23.s, z15.s, z1.s[1]\n"
"ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
@@ -693,14 +726,13 @@
"fmla z22.s, z14.s, z1.s[3]\n"
"fmla z19.s, z15.s, z0.s[3]\n"
"fmla z23.s, z15.s, z1.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z5.s[0]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
"fmla z21.s, z9.s, z5.s[0]\n"
@@ -708,13 +740,13 @@
"fmla z22.s, z10.s, z5.s[0]\n"
"fmla z19.s, z11.s, z4.s[0]\n"
"fmla z23.s, z11.s, z5.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.s, z12.s, z5.s[1]\n"
"fmla z17.s, z13.s, z4.s[1]\n"
"fmla z21.s, z13.s, z5.s[1]\n"
@@ -722,11 +754,12 @@
"fmla z22.s, z14.s, z5.s[1]\n"
"fmla z19.s, z15.s, z4.s[1]\n"
"fmla z23.s, z15.s, z5.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
@@ -735,10 +768,28 @@
"fmla z22.s, z10.s, z5.s[2]\n"
"fmla z19.s, z11.s, z4.s[2]\n"
"fmla z23.s, z11.s, z5.s[2]\n"
- "6:\n"
+ "5:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
+ "fmax z20.s, p7/m, z20.s, z14.s\n"
+ "fmax z21.s, p7/m, z21.s, z14.s\n"
+ "fmax z22.s, p7/m, z22.s, z14.s\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.s, p7/m, z23.s, z14.s\n"
+ "fmin z20.s, p7/m, z20.s, z15.s\n"
+ "fmin z21.s, p7/m, z21.s, z15.s\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.s, p7/m, z22.s, z15.s\n"
+ "fmin z23.s, p7/m, z23.s, z15.s\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
"st1w z20.s, p0, [c_ptr1]\n"
@@ -748,7 +799,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -760,110 +811,97 @@
"c_ptr2 .req X3\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"whilelt p6.s, %[temp], %[leftovers]\n"
"whilelt p0.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
"ptrue p7.s\n"
"whilelt p1.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "mov z20.d, z16.d\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
+ "mov z21.d, z17.d\n"
"ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
+ "mov z22.d, z18.d\n"
"ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
+ "mov z23.d, z19.d\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "mov z20.s, #0\n"
+ "mov z24.d, z16.d\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.s, #0\n"
+ "mov z25.d, z17.d\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z22.s, #0\n"
+ "mov z26.d, z18.d\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z23.s, #0\n"
+ "mov z27.d, z19.d\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z24.s, #0\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z25.s, #0\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z26.s, #0\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z27.s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"add a_ptr1, a_ptr1, #0x10\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "b 2f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z24.s, p0/z, [c_ptr2]\n"
- "fmul z20.s, p7/m, z20.s, z15.s\n"
"ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z21.s, p7/m, z21.s, z15.s\n"
"ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z22.s, p7/m, z22.s, z15.s\n"
"ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z23.s, p7/m, z23.s, z15.s\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "fmul z24.s, p7/m, z24.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "fmul z25.s, p7/m, z25.s, z15.s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "fmul z26.s, p7/m, z26.s, z15.s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmul z27.s, p7/m, z27.s, z15.s\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "2:\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 3f\n"
- "4:\n"
+ "cbz %[loops], 2f\n"
+ "3:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
+ "subs %[loops], %[loops], #0x1\n"
"fmla z22.s, z10.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z26.s, z10.s, z2.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"fmla z23.s, z11.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
@@ -887,8 +925,8 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
@@ -955,8 +993,8 @@
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
"fmla z25.s, z9.s, z6.s[2]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"fmla z18.s, z10.s, z4.s[2]\n"
@@ -982,19 +1020,19 @@
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 4b\n"
- "3:\n"
- "cbz %[regs], 5f\n"
+ "b.ne 3b\n"
+ "2:\n"
+ "cbz %[regs], 4f\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
@@ -1026,8 +1064,8 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
@@ -1059,10 +1097,13 @@
"fmla z16.s, z8.s, z4.s[0]\n"
"ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
"fmla z20.s, z8.s, z5.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z24.s, z8.s, z6.s[0]\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z21.s, z9.s, z5.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z25.s, z9.s, z6.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z4.s[0]\n"
@@ -1090,6 +1131,7 @@
"fmla z27.s, z15.s, z6.s[1]\n"
"ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
@@ -1113,14 +1155,13 @@
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
@@ -1132,13 +1173,13 @@
"fmla z19.s, z11.s, z0.s[0]\n"
"fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
@@ -1150,11 +1191,12 @@
"fmla z19.s, z15.s, z0.s[1]\n"
"fmla z23.s, z15.s, z1.s[1]\n"
"fmla z27.s, z15.s, z2.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
@@ -1167,24 +1209,28 @@
"fmla z19.s, z11.s, z0.s[2]\n"
"fmla z23.s, z11.s, z1.s[2]\n"
"fmla z27.s, z11.s, z2.s[2]\n"
- "b 6f\n"
- "5:\n"
+ "b 5f\n"
+ "4:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1rqw z6.s, p6/z, [a_ptr2]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z22.s, z10.s, z1.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z26.s, z10.s, z2.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
@@ -1205,6 +1251,7 @@
"fmla z27.s, z15.s, z2.s[1]\n"
"ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
@@ -1228,14 +1275,13 @@
"fmla z19.s, z15.s, z0.s[3]\n"
"fmla z23.s, z15.s, z1.s[3]\n"
"fmla z27.s, z15.s, z2.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z5.s[0]\n"
"fmla z24.s, z8.s, z6.s[0]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
@@ -1247,13 +1293,13 @@
"fmla z19.s, z11.s, z4.s[0]\n"
"fmla z23.s, z11.s, z5.s[0]\n"
"fmla z27.s, z11.s, z6.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.s, z12.s, z5.s[1]\n"
"fmla z24.s, z12.s, z6.s[1]\n"
"fmla z17.s, z13.s, z4.s[1]\n"
@@ -1265,11 +1311,12 @@
"fmla z19.s, z15.s, z4.s[1]\n"
"fmla z23.s, z15.s, z5.s[1]\n"
"fmla z27.s, z15.s, z6.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
@@ -1282,14 +1329,40 @@
"fmla z19.s, z11.s, z4.s[2]\n"
"fmla z23.s, z11.s, z5.s[2]\n"
"fmla z27.s, z11.s, z6.s[2]\n"
- "6:\n"
+ "5:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
+ "fmax z20.s, p7/m, z20.s, z14.s\n"
+ "fmax z21.s, p7/m, z21.s, z14.s\n"
+ "fmax z22.s, p7/m, z22.s, z14.s\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.s, p7/m, z23.s, z14.s\n"
+ "fmin z20.s, p7/m, z20.s, z15.s\n"
+ "fmin z21.s, p7/m, z21.s, z15.s\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.s, p7/m, z22.s, z15.s\n"
+ "fmin z23.s, p7/m, z23.s, z15.s\n"
+ "fmax z24.s, p7/m, z24.s, z14.s\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.s, p7/m, z25.s, z14.s\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.s, p7/m, z26.s, z14.s\n"
"st1w z20.s, p0, [c_ptr1]\n"
+ "fmin z24.s, p7/m, z24.s, z15.s\n"
+ "fmin z25.s, p7/m, z25.s, z15.s\n"
+ "fmax z27.s, p7/m, z27.s, z14.s\n"
"st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.s, p7/m, z26.s, z15.s\n"
+ "fmin z27.s, p7/m, z27.s, z15.s\n"
"st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
"st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
"st1w z24.s, p0, [c_ptr2]\n"
@@ -1301,7 +1374,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -1316,132 +1389,115 @@
"c_ptr3 .req X5\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
"whilelt p6.s, %[temp], %[leftovers]\n"
"whilelt p0.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
"ptrue p7.s\n"
"whilelt p1.s, %[temp], %[width]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
"incw %[temp], all, mul #1\n"
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
+ "cbnz %[append], 1f\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "mov z20.d, z16.d\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
+ "mov z21.d, z17.d\n"
"ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
+ "mov z22.d, z18.d\n"
"ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
+ "mov z23.d, z19.d\n"
"ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "mov z20.s, #0\n"
+ "mov z24.d, z16.d\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "mov z21.s, #0\n"
+ "mov z25.d, z17.d\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z22.s, #0\n"
+ "mov z26.d, z18.d\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z23.s, #0\n"
+ "mov z27.d, z19.d\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z24.s, #0\n"
+ "mov z28.d, z16.d\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z25.s, #0\n"
+ "mov z29.d, z17.d\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z26.s, #0\n"
+ "mov z30.d, z18.d\n"
"ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z27.s, #0\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z28.s, #0\n"
+ "mov z31.d, z19.d\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z29.s, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "mov z30.s, #0\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "mov z31.s, #0\n"
"add a_ptr3, a_ptr3, #0x10\n"
- "b 2f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z24.s, p0/z, [c_ptr2]\n"
- "fmul z20.s, p7/m, z20.s, z15.s\n"
"ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z21.s, p7/m, z21.s, z15.s\n"
"ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z22.s, p7/m, z22.s, z15.s\n"
"ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z23.s, p7/m, z23.s, z15.s\n"
"ld1w z28.s, p0/z, [c_ptr3]\n"
- "fmul z24.s, p7/m, z24.s, z15.s\n"
"ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "fmul z25.s, p7/m, z25.s, z15.s\n"
"ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "fmul z26.s, p7/m, z26.s, z15.s\n"
"ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "fmul z27.s, p7/m, z27.s, z15.s\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "fmul z28.s, p7/m, z28.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "fmul z29.s, p7/m, z29.s, z15.s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "fmul z30.s, p7/m, z30.s, z15.s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmul z31.s, p7/m, z31.s, z15.s\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
"ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "2:\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 3f\n"
- "4:\n"
+ "cbz %[loops], 2f\n"
+ "3:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
"fmla z28.s, z8.s, z3.s[0]\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1rqw z7.s, p7/z, [a_ptr3]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
+ "subs %[loops], %[loops], #0x1\n"
"fmla z29.s, z9.s, z3.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z22.s, z10.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"fmla z26.s, z10.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
"fmla z30.s, z10.s, z3.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
"fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"fmla z31.s, z11.s, z3.s[0]\n"
@@ -1471,8 +1527,8 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z28.s, z8.s, z3.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"fmla z29.s, z9.s, z3.s[2]\n"
@@ -1556,8 +1612,8 @@
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
"fmla z28.s, z8.s, z7.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
"fmla z21.s, z9.s, z5.s[2]\n"
"fmla z25.s, z9.s, z6.s[2]\n"
"fmla z29.s, z9.s, z7.s[2]\n"
@@ -1591,21 +1647,21 @@
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
"fmla z31.s, z15.s, z7.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 4b\n"
- "3:\n"
- "cbz %[regs], 5f\n"
+ "b.ne 3b\n"
+ "2:\n"
+ "cbz %[regs], 4f\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
"fmla z28.s, z8.s, z3.s[0]\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1rqw z7.s, p7/z, [a_ptr3]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
"fmla z29.s, z9.s, z3.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1644,8 +1700,8 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z28.s, z8.s, z3.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"fmla z29.s, z9.s, z3.s[2]\n"
@@ -1686,11 +1742,15 @@
"fmla z16.s, z8.s, z4.s[0]\n"
"ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
"fmla z20.s, z8.s, z5.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z24.s, z8.s, z6.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z28.s, z8.s, z7.s[0]\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z21.s, z9.s, z5.s[0]\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"fmla z25.s, z9.s, z6.s[0]\n"
"fmla z29.s, z9.s, z7.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1725,6 +1785,7 @@
"fmla z31.s, z15.s, z7.s[1]\n"
"ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
"fmla z28.s, z8.s, z7.s[2]\n"
@@ -1756,14 +1817,13 @@
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
"fmla z31.s, z15.s, z7.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
"fmla z28.s, z8.s, z3.s[0]\n"
@@ -1779,13 +1839,13 @@
"fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"fmla z31.s, z11.s, z3.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
"fmla z28.s, z12.s, z3.s[1]\n"
@@ -1801,11 +1861,12 @@
"fmla z23.s, z15.s, z1.s[1]\n"
"fmla z27.s, z15.s, z2.s[1]\n"
"fmla z31.s, z15.s, z3.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
@@ -1822,25 +1883,30 @@
"fmla z23.s, z11.s, z1.s[2]\n"
"fmla z27.s, z11.s, z2.s[2]\n"
"fmla z31.s, z11.s, z3.s[2]\n"
- "b 6f\n"
- "5:\n"
+ "b 5f\n"
+ "4:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
"fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
"fmla z28.s, z8.s, z3.s[0]\n"
- "ld1rqw z7.s, p6/z, [a_ptr3]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
"ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z6.s, p6/z, [a_ptr2]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1rqw z7.s, p6/z, [a_ptr3]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z29.s, z9.s, z3.s[0]\n"
"ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z22.s, z10.s, z1.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"fmla z26.s, z10.s, z2.s[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"fmla z30.s, z10.s, z3.s[0]\n"
"ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
@@ -1869,6 +1935,7 @@
"fmla z31.s, z15.s, z3.s[1]\n"
"ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z28.s, z8.s, z3.s[2]\n"
@@ -1900,14 +1967,13 @@
"fmla z23.s, z15.s, z1.s[3]\n"
"fmla z27.s, z15.s, z2.s[3]\n"
"fmla z31.s, z15.s, z3.s[3]\n"
- "cbz %[blocks], 6f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "cbz %[blocks], 5f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z5.s[0]\n"
"fmla z24.s, z8.s, z6.s[0]\n"
"fmla z28.s, z8.s, z7.s[0]\n"
@@ -1923,13 +1989,13 @@
"fmla z23.s, z11.s, z5.s[0]\n"
"fmla z27.s, z11.s, z6.s[0]\n"
"fmla z31.s, z11.s, z7.s[0]\n"
- "b.eq 6f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "b.eq 5f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"fmla z20.s, z12.s, z5.s[1]\n"
"fmla z24.s, z12.s, z6.s[1]\n"
"fmla z28.s, z12.s, z7.s[1]\n"
@@ -1945,11 +2011,12 @@
"fmla z23.s, z15.s, z5.s[1]\n"
"fmla z27.s, z15.s, z6.s[1]\n"
"fmla z31.s, z15.s, z7.s[1]\n"
- "b.eq 6f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "b.eq 5f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
@@ -1966,17 +2033,51 @@
"fmla z23.s, z11.s, z5.s[2]\n"
"fmla z27.s, z11.s, z6.s[2]\n"
"fmla z31.s, z11.s, z7.s[2]\n"
- "6:\n"
+ "5:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
+ "fmax z20.s, p7/m, z20.s, z14.s\n"
+ "fmax z21.s, p7/m, z21.s, z14.s\n"
+ "fmax z22.s, p7/m, z22.s, z14.s\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.s, p7/m, z23.s, z14.s\n"
+ "fmin z20.s, p7/m, z20.s, z15.s\n"
+ "fmin z21.s, p7/m, z21.s, z15.s\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.s, p7/m, z22.s, z15.s\n"
+ "fmin z23.s, p7/m, z23.s, z15.s\n"
+ "fmax z24.s, p7/m, z24.s, z14.s\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.s, p7/m, z25.s, z14.s\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.s, p7/m, z26.s, z14.s\n"
"st1w z20.s, p0, [c_ptr1]\n"
+ "fmin z24.s, p7/m, z24.s, z15.s\n"
+ "fmin z25.s, p7/m, z25.s, z15.s\n"
+ "fmax z27.s, p7/m, z27.s, z14.s\n"
"st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.s, p7/m, z26.s, z15.s\n"
+ "fmax z28.s, p7/m, z28.s, z14.s\n"
+ "fmax z29.s, p7/m, z29.s, z14.s\n"
"st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "fmin z27.s, p7/m, z27.s, z15.s\n"
+ "fmax z30.s, p7/m, z30.s, z14.s\n"
+ "fmin z28.s, p7/m, z28.s, z15.s\n"
"st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "fmin z29.s, p7/m, z29.s, z15.s\n"
+ "fmax z31.s, p7/m, z31.s, z14.s\n"
+ "fmin z30.s, p7/m, z30.s, z15.s\n"
"st1w z24.s, p0, [c_ptr2]\n"
+ "fmin z31.s, p7/m, z31.s, z15.s\n"
"st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
"st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
"st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
@@ -1991,11 +2092,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
index ffd7918..d842210 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,7 +32,7 @@
{
// Actual kernel implementations
-void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
class hybrid_s8s32_dot_4VLx4
{
@@ -40,10 +40,10 @@
typedef int8_t operand_type;
typedef int32_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+ typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -53,20 +53,32 @@
return get_vector_length<int32_t>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
// Default to the generic kernel
kern_type kernel=sve_hybrid_s8s32_dot_4VLx4;
- hybrid_s8s32_dot_4VLx4(const CPUInfo *ci)
- {
-
- }
+ hybrid_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp
index 673f186..d57557a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,14 +25,14 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0);
+void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) {
const int K_stride = ((K + 3) / 4) * 4;
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
@@ -46,17 +46,16 @@
const unsigned long ldab = lda * sizeof(int8_t);
int32_t *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(int32_t);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>()));
- const int32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
long blocks = blocks_count;
const int8_t *a_ptr0 = a_ptr0_base;
const int8_t *b_ptr0 = B + (K_stride * x0);
+ const unsigned long ldcb = ldc * sizeof(int32_t);
switch(M-y) {
case 1:
@@ -70,7 +69,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -84,30 +83,23 @@
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -224,13 +216,12 @@
"sdot z19.s, z11.b, z4.b[0]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[1]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
@@ -241,45 +232,48 @@
"sdot z19.s, z11.b, z4.b[2]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"sdot z17.s, z13.b, z4.b[3]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z18.s, z14.b, z4.b[3]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"sdot z19.s, z15.b, z4.b[3]\n"
"cbz %[blocks], 5f\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "b.eq 5f\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "b.eq 5f\n"
"addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z17.s, z9.b, z0.b[2]\n"
"sdot z18.s, z10.b, z0.b[2]\n"
"sdot z19.s, z11.b, z0.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[3]\n"
"sdot z17.s, z13.b, z0.b[3]\n"
"sdot z18.s, z14.b, z0.b[3]\n"
@@ -289,65 +283,67 @@
"sdot z16.s, z8.b, z0.b[0]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
"sdot z18.s, z10.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z19.s, z11.b, z0.b[2]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"sdot z16.s, z12.b, z0.b[3]\n"
"sdot z17.s, z13.b, z0.b[3]\n"
"sdot z18.s, z14.b, z0.b[3]\n"
"sdot z19.s, z15.b, z0.b[3]\n"
"cbz %[blocks], 5f\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "b.eq 5f\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "b.eq 5f\n"
"addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z17.s, z9.b, z4.b[2]\n"
"sdot z18.s, z10.b, z4.b[2]\n"
"sdot z19.s, z11.b, z4.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[3]\n"
"sdot z17.s, z13.b, z4.b[3]\n"
"sdot z18.s, z14.b, z4.b[3]\n"
@@ -359,7 +355,7 @@
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -378,7 +374,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -397,41 +393,30 @@
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"add a_ptr1, a_ptr1, #0x10\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -601,9 +586,11 @@
"sdot z20.s, z8.b, z5.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"sdot z17.s, z9.b, z4.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"sdot z21.s, z9.b, z5.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z18.s, z10.b, z4.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"sdot z22.s, z10.b, z5.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z19.s, z11.b, z4.b[0]\n"
@@ -622,6 +609,7 @@
"sdot z23.s, z15.b, z5.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z20.s, z8.b, z5.b[2]\n"
"sdot z17.s, z9.b, z4.b[2]\n"
"sdot z21.s, z9.b, z5.b[2]\n"
@@ -638,13 +626,12 @@
"sdot z19.s, z15.b, z4.b[3]\n"
"sdot z23.s, z15.b, z5.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z1.b[0]\n"
"sdot z17.s, z9.b, z0.b[0]\n"
"sdot z21.s, z9.b, z1.b[0]\n"
@@ -653,12 +640,12 @@
"sdot z19.s, z11.b, z0.b[0]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z20.s, z12.b, z1.b[1]\n"
"sdot z17.s, z13.b, z0.b[1]\n"
"sdot z21.s, z13.b, z1.b[1]\n"
@@ -667,12 +654,13 @@
"sdot z19.s, z15.b, z0.b[1]\n"
"sdot z23.s, z15.b, z1.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z1.b[2]\n"
"sdot z17.s, z9.b, z0.b[2]\n"
"sdot z21.s, z9.b, z1.b[2]\n"
@@ -681,10 +669,10 @@
"sdot z19.s, z11.b, z0.b[2]\n"
"sdot z23.s, z11.b, z1.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[3]\n"
"sdot z20.s, z12.b, z1.b[3]\n"
"sdot z17.s, z13.b, z0.b[3]\n"
@@ -698,19 +686,21 @@
"sdot z16.s, z8.b, z0.b[0]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
"sdot z22.s, z10.b, z1.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"sdot z23.s, z11.b, z1.b[0]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[1]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"sdot z20.s, z12.b, z1.b[1]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z17.s, z13.b, z0.b[1]\n"
@@ -723,6 +713,7 @@
"sdot z23.s, z15.b, z1.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z20.s, z8.b, z1.b[2]\n"
"sdot z17.s, z9.b, z0.b[2]\n"
"sdot z21.s, z9.b, z1.b[2]\n"
@@ -739,13 +730,12 @@
"sdot z19.s, z15.b, z0.b[3]\n"
"sdot z23.s, z15.b, z1.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z5.b[0]\n"
"sdot z17.s, z9.b, z4.b[0]\n"
"sdot z21.s, z9.b, z5.b[0]\n"
@@ -754,12 +744,12 @@
"sdot z19.s, z11.b, z4.b[0]\n"
"sdot z23.s, z11.b, z5.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z20.s, z12.b, z5.b[1]\n"
"sdot z17.s, z13.b, z4.b[1]\n"
"sdot z21.s, z13.b, z5.b[1]\n"
@@ -768,12 +758,13 @@
"sdot z19.s, z15.b, z4.b[1]\n"
"sdot z23.s, z15.b, z5.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z5.b[2]\n"
"sdot z17.s, z9.b, z4.b[2]\n"
"sdot z21.s, z9.b, z5.b[2]\n"
@@ -782,10 +773,10 @@
"sdot z19.s, z11.b, z4.b[2]\n"
"sdot z23.s, z11.b, z5.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[3]\n"
"sdot z20.s, z12.b, z5.b[3]\n"
"sdot z17.s, z13.b, z4.b[3]\n"
@@ -807,7 +798,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -830,7 +821,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -852,54 +843,39 @@
"mov z25.s, #0\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"mov z26.s, #0\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "mov z27.s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z27.s, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
"add a_ptr2, a_ptr2, #0x10\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
"ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
"ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
"ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
"ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -1120,10 +1096,13 @@
"sdot z16.s, z8.b, z4.b[0]\n"
"ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
"sdot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"sdot z24.s, z8.b, z6.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"sdot z17.s, z9.b, z4.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"sdot z21.s, z9.b, z5.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"sdot z25.s, z9.b, z6.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z18.s, z10.b, z4.b[0]\n"
@@ -1151,6 +1130,7 @@
"sdot z27.s, z15.b, z6.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z20.s, z8.b, z5.b[2]\n"
"sdot z24.s, z8.b, z6.b[2]\n"
"sdot z17.s, z9.b, z4.b[2]\n"
@@ -1175,13 +1155,12 @@
"sdot z23.s, z15.b, z5.b[3]\n"
"sdot z27.s, z15.b, z6.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z1.b[0]\n"
"sdot z24.s, z8.b, z2.b[0]\n"
"sdot z17.s, z9.b, z0.b[0]\n"
@@ -1194,12 +1173,12 @@
"sdot z23.s, z11.b, z1.b[0]\n"
"sdot z27.s, z11.b, z2.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z20.s, z12.b, z1.b[1]\n"
"sdot z24.s, z12.b, z2.b[1]\n"
"sdot z17.s, z13.b, z0.b[1]\n"
@@ -1212,12 +1191,13 @@
"sdot z23.s, z15.b, z1.b[1]\n"
"sdot z27.s, z15.b, z2.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z1.b[2]\n"
"sdot z24.s, z8.b, z2.b[2]\n"
"sdot z17.s, z9.b, z0.b[2]\n"
@@ -1230,10 +1210,10 @@
"sdot z23.s, z11.b, z1.b[2]\n"
"sdot z27.s, z11.b, z2.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[3]\n"
"sdot z20.s, z12.b, z1.b[3]\n"
"sdot z24.s, z12.b, z2.b[3]\n"
@@ -1253,18 +1233,21 @@
"sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
"sdot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
"sdot z25.s, z9.b, z2.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"sdot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"sdot z26.s, z10.b, z2.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"sdot z23.s, z11.b, z1.b[0]\n"
"sdot z27.s, z11.b, z2.b[0]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
@@ -1285,6 +1268,7 @@
"sdot z27.s, z15.b, z2.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z20.s, z8.b, z1.b[2]\n"
"sdot z24.s, z8.b, z2.b[2]\n"
"sdot z17.s, z9.b, z0.b[2]\n"
@@ -1309,13 +1293,12 @@
"sdot z23.s, z15.b, z1.b[3]\n"
"sdot z27.s, z15.b, z2.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z5.b[0]\n"
"sdot z24.s, z8.b, z6.b[0]\n"
"sdot z17.s, z9.b, z4.b[0]\n"
@@ -1328,12 +1311,12 @@
"sdot z23.s, z11.b, z5.b[0]\n"
"sdot z27.s, z11.b, z6.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z20.s, z12.b, z5.b[1]\n"
"sdot z24.s, z12.b, z6.b[1]\n"
"sdot z17.s, z13.b, z4.b[1]\n"
@@ -1346,12 +1329,13 @@
"sdot z23.s, z15.b, z5.b[1]\n"
"sdot z27.s, z15.b, z6.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z5.b[2]\n"
"sdot z24.s, z8.b, z6.b[2]\n"
"sdot z17.s, z9.b, z4.b[2]\n"
@@ -1364,10 +1348,10 @@
"sdot z23.s, z11.b, z5.b[2]\n"
"sdot z27.s, z11.b, z6.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[3]\n"
"sdot z20.s, z12.b, z5.b[3]\n"
"sdot z24.s, z12.b, z6.b[3]\n"
@@ -1399,7 +1383,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -1427,7 +1411,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -1451,68 +1435,49 @@
"mov z26.s, #0\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"mov z27.s, #0\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "mov z28.s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z29.s, #0\n"
+ "mov z28.s, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "mov z30.s, #0\n"
+ "mov z29.s, #0\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "mov z31.s, #0\n"
+ "mov z30.s, #0\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "mov z31.s, #0\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
"ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
"ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
"ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
"ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
"ld1w z28.s, p0/z, [c_ptr3]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
"ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
"ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
"ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "mul z28.s, p7/m, z28.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z29.s, p7/m, z29.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z30.s, p7/m, z30.s, z15.s\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z31.s, p7/m, z31.s, z15.s\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -1786,11 +1751,15 @@
"sdot z16.s, z8.b, z4.b[0]\n"
"ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
"sdot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"sdot z24.s, z8.b, z6.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"sdot z28.s, z8.b, z7.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"sdot z17.s, z9.b, z4.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"sdot z21.s, z9.b, z5.b[0]\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"sdot z25.s, z9.b, z6.b[0]\n"
"sdot z29.s, z9.b, z7.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1825,6 +1794,7 @@
"sdot z31.s, z15.b, z7.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z20.s, z8.b, z5.b[2]\n"
"sdot z24.s, z8.b, z6.b[2]\n"
"sdot z28.s, z8.b, z7.b[2]\n"
@@ -1857,13 +1827,12 @@
"sdot z27.s, z15.b, z6.b[3]\n"
"sdot z31.s, z15.b, z7.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z1.b[0]\n"
"sdot z24.s, z8.b, z2.b[0]\n"
"sdot z28.s, z8.b, z3.b[0]\n"
@@ -1880,12 +1849,12 @@
"sdot z27.s, z11.b, z2.b[0]\n"
"sdot z31.s, z11.b, z3.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z20.s, z12.b, z1.b[1]\n"
"sdot z24.s, z12.b, z2.b[1]\n"
"sdot z28.s, z12.b, z3.b[1]\n"
@@ -1902,12 +1871,13 @@
"sdot z27.s, z15.b, z2.b[1]\n"
"sdot z31.s, z15.b, z3.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z1.b[2]\n"
"sdot z24.s, z8.b, z2.b[2]\n"
"sdot z28.s, z8.b, z3.b[2]\n"
@@ -1924,10 +1894,10 @@
"sdot z27.s, z11.b, z2.b[2]\n"
"sdot z31.s, z11.b, z3.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z0.b[3]\n"
"sdot z20.s, z12.b, z1.b[3]\n"
"sdot z24.s, z12.b, z2.b[3]\n"
@@ -1953,17 +1923,21 @@
"sdot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
"sdot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr3]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr3]\n"
"sdot z25.s, z9.b, z2.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"sdot z29.s, z9.b, z3.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"sdot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"sdot z26.s, z10.b, z2.b[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"sdot z30.s, z10.b, z3.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
@@ -1992,6 +1966,7 @@
"sdot z31.s, z15.b, z3.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z16.s, z8.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"sdot z20.s, z8.b, z1.b[2]\n"
"sdot z24.s, z8.b, z2.b[2]\n"
"sdot z28.s, z8.b, z3.b[2]\n"
@@ -2024,13 +1999,12 @@
"sdot z27.s, z15.b, z2.b[3]\n"
"sdot z31.s, z15.b, z3.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z5.b[0]\n"
"sdot z24.s, z8.b, z6.b[0]\n"
"sdot z28.s, z8.b, z7.b[0]\n"
@@ -2047,12 +2021,12 @@
"sdot z27.s, z11.b, z6.b[0]\n"
"sdot z31.s, z11.b, z7.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z20.s, z12.b, z5.b[1]\n"
"sdot z24.s, z12.b, z6.b[1]\n"
"sdot z28.s, z12.b, z7.b[1]\n"
@@ -2069,12 +2043,13 @@
"sdot z27.s, z15.b, z6.b[1]\n"
"sdot z31.s, z15.b, z7.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"sdot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z20.s, z8.b, z5.b[2]\n"
"sdot z24.s, z8.b, z6.b[2]\n"
"sdot z28.s, z8.b, z7.b[2]\n"
@@ -2091,10 +2066,10 @@
"sdot z27.s, z11.b, z6.b[2]\n"
"sdot z31.s, z11.b, z7.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"sdot z16.s, z12.b, z4.b[3]\n"
"sdot z20.s, z12.b, z5.b[3]\n"
"sdot z24.s, z12.b, z6.b[3]\n"
@@ -2136,11 +2111,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp
index 2e51349..5dab1da 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,7 +32,7 @@
{
// Actual kernel implementations
-void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
class hybrid_u8u32_dot_4VLx4
{
@@ -40,7 +40,7 @@
typedef uint8_t operand_type;
typedef uint32_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+ typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -53,20 +53,32 @@
return get_vector_length<uint32_t>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
// Default to the generic kernel
kern_type kernel=sve_hybrid_u8u32_dot_4VLx4;
- hybrid_u8u32_dot_4VLx4(const CPUInfo *ci)
- {
-
- }
+ hybrid_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp
index d34d0e5..bf3e8ca 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,14 +25,14 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0u);
+void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) {
const int K_stride = ((K + 3) / 4) * 4;
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
@@ -46,17 +46,16 @@
const unsigned long ldab = lda * sizeof(uint8_t);
uint32_t *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(uint32_t);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>()));
- const uint32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
long blocks = blocks_count;
const uint8_t *a_ptr0 = a_ptr0_base;
const uint8_t *b_ptr0 = B + (K_stride * x0);
+ const unsigned long ldcb = ldc * sizeof(uint32_t);
switch(M-y) {
case 1:
@@ -70,7 +69,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -84,30 +83,23 @@
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -224,13 +216,12 @@
"udot z19.s, z11.b, z4.b[0]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z4.b[1]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
@@ -241,45 +232,48 @@
"udot z19.s, z11.b, z4.b[2]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z16.s, z12.b, z4.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"udot z17.s, z13.b, z4.b[3]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z18.s, z14.b, z4.b[3]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"udot z19.s, z15.b, z4.b[3]\n"
"cbz %[blocks], 5f\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "b.eq 5f\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "b.eq 5f\n"
"addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z17.s, z9.b, z0.b[2]\n"
"udot z18.s, z10.b, z0.b[2]\n"
"udot z19.s, z11.b, z0.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z0.b[3]\n"
"udot z17.s, z13.b, z0.b[3]\n"
"udot z18.s, z14.b, z0.b[3]\n"
@@ -289,65 +283,67 @@
"udot z16.s, z8.b, z0.b[0]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
"udot z18.s, z10.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z19.s, z11.b, z0.b[2]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"udot z16.s, z12.b, z0.b[3]\n"
"udot z17.s, z13.b, z0.b[3]\n"
"udot z18.s, z14.b, z0.b[3]\n"
"udot z19.s, z15.b, z0.b[3]\n"
"cbz %[blocks], 5f\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "b.eq 5f\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "b.eq 5f\n"
"addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z17.s, z9.b, z4.b[2]\n"
"udot z18.s, z10.b, z4.b[2]\n"
"udot z19.s, z11.b, z4.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z4.b[3]\n"
"udot z17.s, z13.b, z4.b[3]\n"
"udot z18.s, z14.b, z4.b[3]\n"
@@ -359,7 +355,7 @@
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -378,7 +374,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -397,41 +393,30 @@
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"add a_ptr1, a_ptr1, #0x10\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -601,9 +586,11 @@
"udot z20.s, z8.b, z5.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"udot z17.s, z9.b, z4.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"udot z21.s, z9.b, z5.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z18.s, z10.b, z4.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"udot z22.s, z10.b, z5.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z19.s, z11.b, z4.b[0]\n"
@@ -622,6 +609,7 @@
"udot z23.s, z15.b, z5.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z20.s, z8.b, z5.b[2]\n"
"udot z17.s, z9.b, z4.b[2]\n"
"udot z21.s, z9.b, z5.b[2]\n"
@@ -638,13 +626,12 @@
"udot z19.s, z15.b, z4.b[3]\n"
"udot z23.s, z15.b, z5.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z1.b[0]\n"
"udot z17.s, z9.b, z0.b[0]\n"
"udot z21.s, z9.b, z1.b[0]\n"
@@ -653,12 +640,12 @@
"udot z19.s, z11.b, z0.b[0]\n"
"udot z23.s, z11.b, z1.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z20.s, z12.b, z1.b[1]\n"
"udot z17.s, z13.b, z0.b[1]\n"
"udot z21.s, z13.b, z1.b[1]\n"
@@ -667,12 +654,13 @@
"udot z19.s, z15.b, z0.b[1]\n"
"udot z23.s, z15.b, z1.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"udot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z1.b[2]\n"
"udot z17.s, z9.b, z0.b[2]\n"
"udot z21.s, z9.b, z1.b[2]\n"
@@ -681,10 +669,10 @@
"udot z19.s, z11.b, z0.b[2]\n"
"udot z23.s, z11.b, z1.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z0.b[3]\n"
"udot z20.s, z12.b, z1.b[3]\n"
"udot z17.s, z13.b, z0.b[3]\n"
@@ -698,19 +686,21 @@
"udot z16.s, z8.b, z0.b[0]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
"udot z22.s, z10.b, z1.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z19.s, z11.b, z0.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"udot z23.s, z11.b, z1.b[0]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z16.s, z12.b, z0.b[1]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"udot z20.s, z12.b, z1.b[1]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z17.s, z13.b, z0.b[1]\n"
@@ -723,6 +713,7 @@
"udot z23.s, z15.b, z1.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z16.s, z8.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z20.s, z8.b, z1.b[2]\n"
"udot z17.s, z9.b, z0.b[2]\n"
"udot z21.s, z9.b, z1.b[2]\n"
@@ -739,13 +730,12 @@
"udot z19.s, z15.b, z0.b[3]\n"
"udot z23.s, z15.b, z1.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z5.b[0]\n"
"udot z17.s, z9.b, z4.b[0]\n"
"udot z21.s, z9.b, z5.b[0]\n"
@@ -754,12 +744,12 @@
"udot z19.s, z11.b, z4.b[0]\n"
"udot z23.s, z11.b, z5.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z20.s, z12.b, z5.b[1]\n"
"udot z17.s, z13.b, z4.b[1]\n"
"udot z21.s, z13.b, z5.b[1]\n"
@@ -768,12 +758,13 @@
"udot z19.s, z15.b, z4.b[1]\n"
"udot z23.s, z15.b, z5.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z5.b[2]\n"
"udot z17.s, z9.b, z4.b[2]\n"
"udot z21.s, z9.b, z5.b[2]\n"
@@ -782,10 +773,10 @@
"udot z19.s, z11.b, z4.b[2]\n"
"udot z23.s, z11.b, z5.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z4.b[3]\n"
"udot z20.s, z12.b, z5.b[3]\n"
"udot z17.s, z13.b, z4.b[3]\n"
@@ -807,7 +798,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -830,7 +821,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -852,54 +843,39 @@
"mov z25.s, #0\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"mov z26.s, #0\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "mov z27.s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z27.s, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
"add a_ptr2, a_ptr2, #0x10\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
"ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
"ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
"ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
"ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -1120,10 +1096,13 @@
"udot z16.s, z8.b, z4.b[0]\n"
"ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
"udot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"udot z24.s, z8.b, z6.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"udot z17.s, z9.b, z4.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"udot z21.s, z9.b, z5.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"udot z25.s, z9.b, z6.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z18.s, z10.b, z4.b[0]\n"
@@ -1151,6 +1130,7 @@
"udot z27.s, z15.b, z6.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z20.s, z8.b, z5.b[2]\n"
"udot z24.s, z8.b, z6.b[2]\n"
"udot z17.s, z9.b, z4.b[2]\n"
@@ -1175,13 +1155,12 @@
"udot z23.s, z15.b, z5.b[3]\n"
"udot z27.s, z15.b, z6.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z1.b[0]\n"
"udot z24.s, z8.b, z2.b[0]\n"
"udot z17.s, z9.b, z0.b[0]\n"
@@ -1194,12 +1173,12 @@
"udot z23.s, z11.b, z1.b[0]\n"
"udot z27.s, z11.b, z2.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z20.s, z12.b, z1.b[1]\n"
"udot z24.s, z12.b, z2.b[1]\n"
"udot z17.s, z13.b, z0.b[1]\n"
@@ -1212,12 +1191,13 @@
"udot z23.s, z15.b, z1.b[1]\n"
"udot z27.s, z15.b, z2.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"udot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z1.b[2]\n"
"udot z24.s, z8.b, z2.b[2]\n"
"udot z17.s, z9.b, z0.b[2]\n"
@@ -1230,10 +1210,10 @@
"udot z23.s, z11.b, z1.b[2]\n"
"udot z27.s, z11.b, z2.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z0.b[3]\n"
"udot z20.s, z12.b, z1.b[3]\n"
"udot z24.s, z12.b, z2.b[3]\n"
@@ -1253,18 +1233,21 @@
"udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
"udot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
"udot z25.s, z9.b, z2.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z18.s, z10.b, z0.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"udot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"udot z26.s, z10.b, z2.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z19.s, z11.b, z0.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"udot z23.s, z11.b, z1.b[0]\n"
"udot z27.s, z11.b, z2.b[0]\n"
"ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
@@ -1285,6 +1268,7 @@
"udot z27.s, z15.b, z2.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z16.s, z8.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z20.s, z8.b, z1.b[2]\n"
"udot z24.s, z8.b, z2.b[2]\n"
"udot z17.s, z9.b, z0.b[2]\n"
@@ -1309,13 +1293,12 @@
"udot z23.s, z15.b, z1.b[3]\n"
"udot z27.s, z15.b, z2.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z5.b[0]\n"
"udot z24.s, z8.b, z6.b[0]\n"
"udot z17.s, z9.b, z4.b[0]\n"
@@ -1328,12 +1311,12 @@
"udot z23.s, z11.b, z5.b[0]\n"
"udot z27.s, z11.b, z6.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z20.s, z12.b, z5.b[1]\n"
"udot z24.s, z12.b, z6.b[1]\n"
"udot z17.s, z13.b, z4.b[1]\n"
@@ -1346,12 +1329,13 @@
"udot z23.s, z15.b, z5.b[1]\n"
"udot z27.s, z15.b, z6.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z5.b[2]\n"
"udot z24.s, z8.b, z6.b[2]\n"
"udot z17.s, z9.b, z4.b[2]\n"
@@ -1364,10 +1348,10 @@
"udot z23.s, z11.b, z5.b[2]\n"
"udot z27.s, z11.b, z6.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z4.b[3]\n"
"udot z20.s, z12.b, z5.b[3]\n"
"udot z24.s, z12.b, z6.b[3]\n"
@@ -1399,7 +1383,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -1427,7 +1411,7 @@
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
+ "cbnz %[append], 1f\n"
"mov z16.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"mov z17.s, #0\n"
@@ -1451,68 +1435,49 @@
"mov z26.s, #0\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"mov z27.s, #0\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "mov z28.s, #0\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z29.s, #0\n"
+ "mov z28.s, #0\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "mov z30.s, #0\n"
+ "mov z29.s, #0\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "mov z31.s, #0\n"
+ "mov z30.s, #0\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "mov z31.s, #0\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"b 3f\n"
"1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
"ld1w z16.s, p0/z, [%[c_ptr0]]\n"
"ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
"ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
"ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
"ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
"ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
"ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
"ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
"ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
"ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
"ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
"ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
"ld1w z28.s, p0/z, [c_ptr3]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
"ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
"ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
"ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "mul z28.s, p7/m, z28.s, z15.s\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z29.s, p7/m, z29.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z30.s, p7/m, z30.s, z15.s\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z31.s, p7/m, z31.s, z15.s\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
"cbz %[loops], 2f\n"
"3:\n"
@@ -1786,11 +1751,15 @@
"udot z16.s, z8.b, z4.b[0]\n"
"ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
"udot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"udot z24.s, z8.b, z6.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"udot z28.s, z8.b, z7.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"udot z17.s, z9.b, z4.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"udot z21.s, z9.b, z5.b[0]\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"udot z25.s, z9.b, z6.b[0]\n"
"udot z29.s, z9.b, z7.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1825,6 +1794,7 @@
"udot z31.s, z15.b, z7.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z20.s, z8.b, z5.b[2]\n"
"udot z24.s, z8.b, z6.b[2]\n"
"udot z28.s, z8.b, z7.b[2]\n"
@@ -1857,13 +1827,12 @@
"udot z27.s, z15.b, z6.b[3]\n"
"udot z31.s, z15.b, z7.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z1.b[0]\n"
"udot z24.s, z8.b, z2.b[0]\n"
"udot z28.s, z8.b, z3.b[0]\n"
@@ -1880,12 +1849,12 @@
"udot z27.s, z11.b, z2.b[0]\n"
"udot z31.s, z11.b, z3.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z20.s, z12.b, z1.b[1]\n"
"udot z24.s, z12.b, z2.b[1]\n"
"udot z28.s, z12.b, z3.b[1]\n"
@@ -1902,12 +1871,13 @@
"udot z27.s, z15.b, z2.b[1]\n"
"udot z31.s, z15.b, z3.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"udot z16.s, z8.b, z0.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z1.b[2]\n"
"udot z24.s, z8.b, z2.b[2]\n"
"udot z28.s, z8.b, z3.b[2]\n"
@@ -1924,10 +1894,10 @@
"udot z27.s, z11.b, z2.b[2]\n"
"udot z31.s, z11.b, z3.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z0.b[3]\n"
"udot z20.s, z12.b, z1.b[3]\n"
"udot z24.s, z12.b, z2.b[3]\n"
@@ -1953,17 +1923,21 @@
"udot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
"udot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr3]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
"ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr3]\n"
"udot z25.s, z9.b, z2.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"udot z29.s, z9.b, z3.b[0]\n"
"ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z18.s, z10.b, z0.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"udot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"udot z26.s, z10.b, z2.b[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"udot z30.s, z10.b, z3.b[0]\n"
"ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z19.s, z11.b, z0.b[0]\n"
@@ -1992,6 +1966,7 @@
"udot z31.s, z15.b, z3.b[1]\n"
"ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z16.s, z8.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
"udot z20.s, z8.b, z1.b[2]\n"
"udot z24.s, z8.b, z2.b[2]\n"
"udot z28.s, z8.b, z3.b[2]\n"
@@ -2024,13 +1999,12 @@
"udot z27.s, z15.b, z2.b[3]\n"
"udot z31.s, z15.b, z3.b[3]\n"
"cbz %[blocks], 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z16.s, z8.b, z4.b[0]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z5.b[0]\n"
"udot z24.s, z8.b, z6.b[0]\n"
"udot z28.s, z8.b, z7.b[0]\n"
@@ -2047,12 +2021,12 @@
"udot z27.s, z11.b, z6.b[0]\n"
"udot z31.s, z11.b, z7.b[0]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z20.s, z12.b, z5.b[1]\n"
"udot z24.s, z12.b, z6.b[1]\n"
"udot z28.s, z12.b, z7.b[1]\n"
@@ -2069,12 +2043,13 @@
"udot z27.s, z15.b, z6.b[1]\n"
"udot z31.s, z15.b, z7.b[1]\n"
"b.eq 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
"subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
"udot z16.s, z8.b, z4.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z20.s, z8.b, z5.b[2]\n"
"udot z24.s, z8.b, z6.b[2]\n"
"udot z28.s, z8.b, z7.b[2]\n"
@@ -2091,10 +2066,10 @@
"udot z27.s, z11.b, z6.b[2]\n"
"udot z31.s, z11.b, z7.b[2]\n"
"b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
"udot z16.s, z12.b, z4.b[3]\n"
"udot z20.s, z12.b, z5.b[3]\n"
"udot z24.s, z12.b, z6.b[3]\n"
@@ -2136,11 +2111,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
index 2ca4ce2..a74f9ef 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
@@ -61,10 +61,7 @@
kern_type kernel=sve_interleaved_fp16_mla_3VLx8;
- interleaved_fp16_mla_3VLx8(const CPUInfo *ci)
- {
-
- }
+ interleaved_fp16_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
index 8c1fe6d..1e73d8e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
@@ -61,10 +61,7 @@
kern_type kernel=sve_interleaved_fp32_mla_3VLx8;
- interleaved_fp32_mla_3VLx8(const CPUInfo *ci)
- {
-
- }
+ interleaved_fp32_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
index cbb2138..8668ea7 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
@@ -61,10 +61,7 @@
kern_type kernel=sve_interleaved_s8s32_dot_3VLx8;
- interleaved_s8s32_dot_3VLx8(const CPUInfo *ci)
- {
-
- }
+ interleaved_s8s32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
index 99c039e..d3c8851 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2019 Arm Limited.
+ * Copyright (c) 2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -61,10 +61,7 @@
kern_type kernel=sve_interleaved_u8u32_dot_3VLx8;
- interleaved_u8u32_dot_3VLx8(const CPUInfo *ci)
- {
-
- }
+ interleaved_u8u32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp
index 6cce601..741f200 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,14 +25,11 @@
#ifdef __ARM_FEATURE_SVE
-
-
-
namespace arm_gemm
{
// Actual kernel implementations
-void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int);
+void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool);
class native_fp16_mla_4VLx4
{
@@ -40,10 +37,10 @@
typedef __fp16 operand_type;
typedef __fp16 result_type;
- typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int);
+ typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -53,20 +50,32 @@
return get_vector_length<__fp16>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
// Default to the generic kernel
kern_type kernel=sve_native_fp16_mla_4VLx4;
- native_fp16_mla_4VLx4(const CPUInfo *ci)
- {
-
- }
+ native_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp
index f1aaeb1..14dd38b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,30 +25,49 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
+void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) {
const long loops_count = ((K + 8) / 16) - 1;
K -= loops_count * 16;
const long regs_count = (K / 8) - 1;
K -= (regs_count + 1) * 8;
const long leftovers = K;
+ __fp16 nullbias[512];
+ if (!append && !bias) {
+ memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16)));
+ }
+ __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ const __fp16 * const minptr = &minval;
+ const __fp16 * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<__fp16>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
for (int y=0; y<M; y+=4) {
const __fp16 * const a_ptr0_base = A + (y * lda);
const unsigned long ldab = lda * sizeof(__fp16);
__fp16 *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(__fp16);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<__fp16>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>()));
- const __fp16 *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
@@ -56,6 +75,8 @@
const __fp16 *a_ptr0 = a_ptr0_base;
const __fp16 *b_ptr0 = B + x0;
long ldbb = ldb * sizeof(__fp16);
+ const unsigned long ldcb = ldc * sizeof(__fp16);
+ const __fp16 *biasptr = bias ? bias+x0 : nullbias;
switch(M-y) {
case 1:
@@ -64,52 +85,34 @@
"whilelt p0.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"ptrue p7.h\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
"whilelt p1.h, %[temp], %[width]\n"
+ "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
"inch %[temp], all, mul #1\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
"whilelt p2.h, %[temp], %[width]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"inch %[temp], all, mul #1\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "mov z18.h, #0\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z19.h, #0\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "cbz %[loops], 2f\n"
- "3:\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
+ "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z17.h, z9.h, z0.h[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"subs %[loops], %[loops], #0x1\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z16.h, z12.h, z0.h[1]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
@@ -119,70 +122,69 @@
"fmla z19.h, z15.h, z0.h[1]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z9.h, z0.h[2]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[2]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[2]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z17.h, z13.h, z0.h[3]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z18.h, z14.h, z0.h[3]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z19.h, z15.h, z0.h[3]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z8.h, z0.h[4]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z9.h, z0.h[4]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[4]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[4]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z17.h, z13.h, z0.h[5]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
+ "fmla z18.h, z14.h, z0.h[5]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
+ "fmla z19.h, z15.h, z0.h[5]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.h, z8.h, z0.h[6]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.h, z9.h, z0.h[6]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.h, z10.h, z0.h[6]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.h, z11.h, z0.h[6]\n"
+ "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.h, z12.h, z0.h[7]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z17.h, z13.h, z0.h[7]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.h, z14.h, z0.h[7]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.h, z15.h, z0.h[7]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z8.h, z4.h[0]\n"
"ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
+ "fmla z18.h, z10.h, z4.h[0]\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.h, z11.h, z4.h[0]\n"
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
@@ -247,94 +249,95 @@
"fmla z18.h, z14.h, z4.h[7]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z15.h, z4.h[7]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
+ "cbz %[regs], 3f\n"
+ "fmla z16.h, z8.h, z0.h[0]\n"
"ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z18.h, z14.h, z0.h[1]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z19.h, z15.h, z0.h[1]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z17.h, z9.h, z0.h[2]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[2]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[2]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z12.h, z0.h[3]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z13.h, z0.h[3]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z14.h, z0.h[3]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z15.h, z0.h[3]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z17.h, z9.h, z0.h[4]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"fmla z18.h, z10.h, z0.h[4]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[4]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z12.h, z0.h[5]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z13.h, z0.h[5]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z14.h, z0.h[5]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z15.h, z0.h[5]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z8.h, z0.h[6]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z17.h, z9.h, z0.h[6]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
+ "fmla z18.h, z10.h, z0.h[6]\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
+ "fmla z19.h, z11.h, z0.h[6]\n"
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.h, z12.h, z0.h[7]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.h, z13.h, z0.h[7]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.h, z14.h, z0.h[7]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.h, z15.h, z0.h[7]\n"
+ "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.h, z8.h, z4.h[0]\n"
"ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z17.h, z9.h, z4.h[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.h, z10.h, z4.h[0]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.h, z11.h, z4.h[0]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
+ "fmla z16.h, z12.h, z4.h[1]\n"
+ "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z17.h, z13.h, z4.h[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.h, z14.h, z4.h[1]\n"
@@ -392,7 +395,7 @@
"fmla z17.h, z13.h, z4.h[7]\n"
"fmla z18.h, z14.h, z4.h[7]\n"
"fmla z19.h, z15.h, z4.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -403,7 +406,7 @@
"fmla z17.h, z9.h, z0.h[0]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -414,7 +417,7 @@
"fmla z17.h, z13.h, z0.h[1]\n"
"fmla z18.h, z14.h, z0.h[1]\n"
"fmla z19.h, z15.h, z0.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -425,7 +428,7 @@
"fmla z17.h, z9.h, z0.h[2]\n"
"fmla z18.h, z10.h, z0.h[2]\n"
"fmla z19.h, z11.h, z0.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -436,7 +439,7 @@
"fmla z17.h, z13.h, z0.h[3]\n"
"fmla z18.h, z14.h, z0.h[3]\n"
"fmla z19.h, z15.h, z0.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -447,7 +450,7 @@
"fmla z17.h, z9.h, z0.h[4]\n"
"fmla z18.h, z10.h, z0.h[4]\n"
"fmla z19.h, z11.h, z0.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -458,7 +461,7 @@
"fmla z17.h, z13.h, z0.h[5]\n"
"fmla z18.h, z14.h, z0.h[5]\n"
"fmla z19.h, z15.h, z0.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -468,73 +471,73 @@
"fmla z17.h, z9.h, z0.h[6]\n"
"fmla z18.h, z10.h, z0.h[6]\n"
"fmla z19.h, z11.h, z0.h[6]\n"
- "b 5f\n"
- "4:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
"ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z18.h, z14.h, z0.h[1]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z19.h, z15.h, z0.h[1]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z17.h, z9.h, z0.h[2]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[2]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[2]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z12.h, z0.h[3]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z13.h, z0.h[3]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z14.h, z0.h[3]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z15.h, z0.h[3]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z17.h, z9.h, z0.h[4]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
+ "fmla z18.h, z10.h, z0.h[4]\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
+ "fmla z19.h, z11.h, z0.h[4]\n"
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.h, z12.h, z0.h[5]\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.h, z13.h, z0.h[5]\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.h, z14.h, z0.h[5]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.h, z15.h, z0.h[5]\n"
+ "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.h, z8.h, z0.h[6]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z17.h, z9.h, z0.h[6]\n"
+ "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.h, z10.h, z0.h[6]\n"
+ "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.h, z11.h, z0.h[6]\n"
+ "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z16.h, z12.h, z0.h[7]\n"
"fmla z17.h, z13.h, z0.h[7]\n"
"fmla z18.h, z14.h, z0.h[7]\n"
"fmla z19.h, z15.h, z0.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -545,7 +548,7 @@
"fmla z17.h, z9.h, z4.h[0]\n"
"fmla z18.h, z10.h, z4.h[0]\n"
"fmla z19.h, z11.h, z4.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -556,7 +559,7 @@
"fmla z17.h, z13.h, z4.h[1]\n"
"fmla z18.h, z14.h, z4.h[1]\n"
"fmla z19.h, z15.h, z4.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -567,7 +570,7 @@
"fmla z17.h, z9.h, z4.h[2]\n"
"fmla z18.h, z10.h, z4.h[2]\n"
"fmla z19.h, z11.h, z4.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -578,7 +581,7 @@
"fmla z17.h, z13.h, z4.h[3]\n"
"fmla z18.h, z14.h, z4.h[3]\n"
"fmla z19.h, z15.h, z4.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -589,7 +592,7 @@
"fmla z17.h, z9.h, z4.h[4]\n"
"fmla z18.h, z10.h, z4.h[4]\n"
"fmla z19.h, z11.h, z4.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -600,7 +603,7 @@
"fmla z17.h, z13.h, z4.h[5]\n"
"fmla z18.h, z14.h, z4.h[5]\n"
"fmla z19.h, z15.h, z4.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -610,14 +613,24 @@
"fmla z17.h, z9.h, z4.h[6]\n"
"fmla z18.h, z10.h, z4.h[6]\n"
"fmla z19.h, z11.h, z4.h[6]\n"
- "5:\n"
+ "4:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -631,66 +644,33 @@
"whilelt p0.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"ptrue p7.h\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
"whilelt p1.h, %[temp], %[width]\n"
+ "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
"inch %[temp], all, mul #1\n"
+ "mov z20.d, z16.d\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1rqh z1.h, p7/z, [a_ptr1]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"whilelt p2.h, %[temp], %[width]\n"
+ "mov z21.d, z17.d\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"inch %[temp], all, mul #1\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z18.h, #0\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "mov z19.h, #0\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.h, #0\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z21.h, #0\n"
+ "mov z22.d, z18.d\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.h, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z23.h, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z23.d, z19.d\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
- "ld1h z20.h, p0/z, [c_ptr1]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
- "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
- "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
- "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z20.h, p7/m, z20.h, z15.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z21.h, p7/m, z21.h, z15.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "fmul z22.h, p7/m, z22.h, z15.h\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmul z23.h, p7/m, z23.h, z15.h\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[0]\n"
@@ -906,23 +886,23 @@
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z15.h, z4.h[7]\n"
"fmla z23.h, z15.h, z5.h[7]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
+ "cbz %[regs], 3f\n"
+ "fmla z16.h, z8.h, z0.h[0]\n"
"ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
+ "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
+ "fmla z18.h, z10.h, z0.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
+ "fmla z22.h, z10.h, z1.h[0]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.h, z11.h, z0.h[0]\n"
"fmla z23.h, z11.h, z1.h[0]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
@@ -1026,9 +1006,11 @@
"fmla z21.h, z9.h, z5.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z4.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z22.h, z10.h, z5.h[0]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z4.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z23.h, z11.h, z5.h[0]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.h, z12.h, z4.h[1]\n"
@@ -1112,7 +1094,7 @@
"fmla z22.h, z14.h, z5.h[7]\n"
"fmla z19.h, z15.h, z4.h[7]\n"
"fmla z23.h, z15.h, z5.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -1127,7 +1109,7 @@
"fmla z22.h, z10.h, z1.h[0]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
"fmla z23.h, z11.h, z1.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -1142,7 +1124,7 @@
"fmla z22.h, z14.h, z1.h[1]\n"
"fmla z19.h, z15.h, z0.h[1]\n"
"fmla z23.h, z15.h, z1.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -1157,7 +1139,7 @@
"fmla z22.h, z10.h, z1.h[2]\n"
"fmla z19.h, z11.h, z0.h[2]\n"
"fmla z23.h, z11.h, z1.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -1172,7 +1154,7 @@
"fmla z22.h, z14.h, z1.h[3]\n"
"fmla z19.h, z15.h, z0.h[3]\n"
"fmla z23.h, z15.h, z1.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -1187,7 +1169,7 @@
"fmla z22.h, z10.h, z1.h[4]\n"
"fmla z19.h, z11.h, z0.h[4]\n"
"fmla z23.h, z11.h, z1.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -1202,7 +1184,7 @@
"fmla z22.h, z14.h, z1.h[5]\n"
"fmla z19.h, z15.h, z0.h[5]\n"
"fmla z23.h, z15.h, z1.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1216,22 +1198,22 @@
"fmla z22.h, z10.h, z1.h[6]\n"
"fmla z19.h, z11.h, z0.h[6]\n"
"fmla z23.h, z11.h, z1.h[6]\n"
- "b 5f\n"
- "4:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
+ "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
+ "fmla z18.h, z10.h, z0.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
+ "fmla z22.h, z10.h, z1.h[0]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.h, z11.h, z0.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z23.h, z11.h, z1.h[0]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
@@ -1239,6 +1221,7 @@
"fmla z20.h, z12.h, z1.h[1]\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z21.h, z13.h, z1.h[1]\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z14.h, z0.h[1]\n"
@@ -1315,7 +1298,7 @@
"fmla z22.h, z14.h, z1.h[7]\n"
"fmla z19.h, z15.h, z0.h[7]\n"
"fmla z23.h, z15.h, z1.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -1330,7 +1313,7 @@
"fmla z22.h, z10.h, z5.h[0]\n"
"fmla z19.h, z11.h, z4.h[0]\n"
"fmla z23.h, z11.h, z5.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -1345,7 +1328,7 @@
"fmla z22.h, z14.h, z5.h[1]\n"
"fmla z19.h, z15.h, z4.h[1]\n"
"fmla z23.h, z15.h, z5.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -1360,7 +1343,7 @@
"fmla z22.h, z10.h, z5.h[2]\n"
"fmla z19.h, z11.h, z4.h[2]\n"
"fmla z23.h, z11.h, z5.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -1375,7 +1358,7 @@
"fmla z22.h, z14.h, z5.h[3]\n"
"fmla z19.h, z15.h, z4.h[3]\n"
"fmla z23.h, z15.h, z5.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -1390,7 +1373,7 @@
"fmla z22.h, z10.h, z5.h[4]\n"
"fmla z19.h, z11.h, z4.h[4]\n"
"fmla z23.h, z11.h, z5.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -1405,7 +1388,7 @@
"fmla z22.h, z14.h, z5.h[5]\n"
"fmla z19.h, z15.h, z4.h[5]\n"
"fmla z23.h, z15.h, z5.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1419,10 +1402,28 @@
"fmla z22.h, z10.h, z5.h[6]\n"
"fmla z19.h, z11.h, z4.h[6]\n"
"fmla z23.h, z11.h, z5.h[6]\n"
- "5:\n"
+ "4:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
+ "fmax z20.h, p7/m, z20.h, z14.h\n"
+ "fmax z21.h, p7/m, z21.h, z14.h\n"
+ "fmax z22.h, p7/m, z22.h, z14.h\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.h, p7/m, z23.h, z14.h\n"
+ "fmin z20.h, p7/m, z20.h, z15.h\n"
+ "fmin z21.h, p7/m, z21.h, z15.h\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.h, p7/m, z22.h, z15.h\n"
+ "fmin z23.h, p7/m, z23.h, z15.h\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
"st1h z20.h, p0, [c_ptr1]\n"
@@ -1432,7 +1433,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -1450,82 +1451,39 @@
"whilelt p0.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"ptrue p7.h\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
"whilelt p1.h, %[temp], %[width]\n"
+ "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
"inch %[temp], all, mul #1\n"
+ "mov z20.d, z16.d\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "mov z24.d, z16.d\n"
+ "ld1rqh z1.h, p7/z, [a_ptr1]\n"
+ "ld1rqh z2.h, p7/z, [a_ptr2]\n"
"whilelt p2.h, %[temp], %[width]\n"
+ "mov z21.d, z17.d\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "mov z25.d, z17.d\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"inch %[temp], all, mul #1\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z18.h, #0\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z19.h, #0\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "mov z20.h, #0\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.h, #0\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z22.h, #0\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z23.h, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z24.h, #0\n"
+ "mov z22.d, z18.d\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "mov z25.h, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "mov z26.h, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "mov z27.h, #0\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
- "ld1h z20.h, p0/z, [c_ptr1]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
- "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
- "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
- "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z20.h, p7/m, z20.h, z15.h\n"
- "ld1h z24.h, p0/z, [c_ptr2]\n"
- "fmul z21.h, p7/m, z21.h, z15.h\n"
- "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z22.h, p7/m, z22.h, z15.h\n"
- "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z23.h, p7/m, z23.h, z15.h\n"
- "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z24.h, p7/m, z24.h, z15.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z25.h, p7/m, z25.h, z15.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "fmul z26.h, p7/m, z26.h, z15.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "fmul z27.h, p7/m, z27.h, z15.h\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z26.d, z18.d\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add a_ptr2, a_ptr2, #0x10\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z23.d, z19.d\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "mov z27.d, z19.d\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[0]\n"
@@ -1551,9 +1509,9 @@
"fmla z27.h, z11.h, z2.h[0]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.h, z12.h, z0.h[1]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
"add a_ptr2, a_ptr2, #0x20\n"
+ "fmla z20.h, z12.h, z1.h[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z24.h, z12.h, z2.h[1]\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z13.h, z0.h[1]\n"
@@ -1808,23 +1766,23 @@
"fmla z19.h, z15.h, z4.h[7]\n"
"fmla z23.h, z15.h, z5.h[7]\n"
"fmla z27.h, z15.h, z6.h[7]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
+ "cbz %[regs], 3f\n"
+ "fmla z16.h, z8.h, z0.h[0]\n"
"ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
+ "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
+ "fmla z24.h, z8.h, z2.h[0]\n"
"ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
+ "fmla z25.h, z9.h, z2.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.h, z10.h, z0.h[0]\n"
"fmla z22.h, z10.h, z1.h[0]\n"
"fmla z26.h, z10.h, z2.h[0]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
@@ -1960,10 +1918,13 @@
"fmla z24.h, z8.h, z6.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z21.h, z9.h, z5.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z25.h, z9.h, z6.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z4.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z22.h, z10.h, z5.h[0]\n"
"fmla z26.h, z10.h, z6.h[0]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
@@ -2080,7 +2041,7 @@
"fmla z19.h, z15.h, z4.h[7]\n"
"fmla z23.h, z15.h, z5.h[7]\n"
"fmla z27.h, z15.h, z6.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -2099,7 +2060,7 @@
"fmla z19.h, z11.h, z0.h[0]\n"
"fmla z23.h, z11.h, z1.h[0]\n"
"fmla z27.h, z11.h, z2.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -2118,7 +2079,7 @@
"fmla z19.h, z15.h, z0.h[1]\n"
"fmla z23.h, z15.h, z1.h[1]\n"
"fmla z27.h, z15.h, z2.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -2137,7 +2098,7 @@
"fmla z19.h, z11.h, z0.h[2]\n"
"fmla z23.h, z11.h, z1.h[2]\n"
"fmla z27.h, z11.h, z2.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -2156,7 +2117,7 @@
"fmla z19.h, z15.h, z0.h[3]\n"
"fmla z23.h, z15.h, z1.h[3]\n"
"fmla z27.h, z15.h, z2.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -2175,7 +2136,7 @@
"fmla z19.h, z11.h, z0.h[4]\n"
"fmla z23.h, z11.h, z1.h[4]\n"
"fmla z27.h, z11.h, z2.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -2194,7 +2155,7 @@
"fmla z19.h, z15.h, z0.h[5]\n"
"fmla z23.h, z15.h, z1.h[5]\n"
"fmla z27.h, z15.h, z2.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -2212,26 +2173,28 @@
"fmla z19.h, z11.h, z0.h[6]\n"
"fmla z23.h, z11.h, z1.h[6]\n"
"fmla z27.h, z11.h, z2.h[6]\n"
- "b 5f\n"
- "4:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
+ "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
+ "fmla z24.h, z8.h, z2.h[0]\n"
"ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
+ "fmla z25.h, z9.h, z2.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.h, z10.h, z0.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z22.h, z10.h, z1.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z26.h, z10.h, z2.h[0]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"fmla z23.h, z11.h, z1.h[0]\n"
"fmla z27.h, z11.h, z2.h[0]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
@@ -2344,7 +2307,7 @@
"fmla z19.h, z15.h, z0.h[7]\n"
"fmla z23.h, z15.h, z1.h[7]\n"
"fmla z27.h, z15.h, z2.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -2363,7 +2326,7 @@
"fmla z19.h, z11.h, z4.h[0]\n"
"fmla z23.h, z11.h, z5.h[0]\n"
"fmla z27.h, z11.h, z6.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -2382,7 +2345,7 @@
"fmla z19.h, z15.h, z4.h[1]\n"
"fmla z23.h, z15.h, z5.h[1]\n"
"fmla z27.h, z15.h, z6.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -2401,7 +2364,7 @@
"fmla z19.h, z11.h, z4.h[2]\n"
"fmla z23.h, z11.h, z5.h[2]\n"
"fmla z27.h, z11.h, z6.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -2420,7 +2383,7 @@
"fmla z19.h, z15.h, z4.h[3]\n"
"fmla z23.h, z15.h, z5.h[3]\n"
"fmla z27.h, z15.h, z6.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -2439,7 +2402,7 @@
"fmla z19.h, z11.h, z4.h[4]\n"
"fmla z23.h, z11.h, z5.h[4]\n"
"fmla z27.h, z11.h, z6.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -2458,7 +2421,7 @@
"fmla z19.h, z15.h, z4.h[5]\n"
"fmla z23.h, z15.h, z5.h[5]\n"
"fmla z27.h, z15.h, z6.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -2476,14 +2439,40 @@
"fmla z19.h, z11.h, z4.h[6]\n"
"fmla z23.h, z11.h, z5.h[6]\n"
"fmla z27.h, z11.h, z6.h[6]\n"
- "5:\n"
+ "4:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
+ "fmax z20.h, p7/m, z20.h, z14.h\n"
+ "fmax z21.h, p7/m, z21.h, z14.h\n"
+ "fmax z22.h, p7/m, z22.h, z14.h\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.h, p7/m, z23.h, z14.h\n"
+ "fmin z20.h, p7/m, z20.h, z15.h\n"
+ "fmin z21.h, p7/m, z21.h, z15.h\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.h, p7/m, z22.h, z15.h\n"
+ "fmin z23.h, p7/m, z23.h, z15.h\n"
+ "fmax z24.h, p7/m, z24.h, z14.h\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.h, p7/m, z25.h, z14.h\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.h, p7/m, z26.h, z14.h\n"
"st1h z20.h, p0, [c_ptr1]\n"
+ "fmin z24.h, p7/m, z24.h, z15.h\n"
+ "fmin z25.h, p7/m, z25.h, z15.h\n"
+ "fmax z27.h, p7/m, z27.h, z14.h\n"
"st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.h, p7/m, z26.h, z15.h\n"
+ "fmin z27.h, p7/m, z27.h, z15.h\n"
"st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n"
"st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n"
"st1h z24.h, p0, [c_ptr2]\n"
@@ -2495,7 +2484,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -2518,98 +2507,45 @@
"whilelt p0.h, %[temp], %[width]\n"
"inch %[temp], all, mul #1\n"
"ptrue p7.h\n"
+ "ld1h z16.h, p0/z, [%[biasptr]]\n"
"whilelt p1.h, %[temp], %[width]\n"
+ "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
"inch %[temp], all, mul #1\n"
+ "mov z20.d, z16.d\n"
+ "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "mov z24.d, z16.d\n"
+ "ld1rqh z1.h, p7/z, [a_ptr1]\n"
+ "mov z28.d, z16.d\n"
+ "ld1rqh z2.h, p7/z, [a_ptr2]\n"
+ "ld1rqh z3.h, p7/z, [a_ptr3]\n"
"whilelt p2.h, %[temp], %[width]\n"
+ "mov z21.d, z17.d\n"
+ "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "mov z25.d, z17.d\n"
+ "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z29.d, z17.d\n"
+ "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"inch %[temp], all, mul #1\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z22.d, z18.d\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z26.d, z18.d\n"
"whilelt p3.h, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.h, #0\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z17.h, #0\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z18.h, #0\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z19.h, #0\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "mov z20.h, #0\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "mov z21.h, #0\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z22.h, #0\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z23.h, #0\n"
- "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z24.h, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z25.h, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "mov z26.h, #0\n"
+ "mov z30.d, z18.d\n"
"add a_ptr2, a_ptr2, #0x10\n"
- "mov z27.h, #0\n"
"add a_ptr3, a_ptr3, #0x10\n"
- "mov z28.h, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "mov z29.h, #0\n"
- "ld1h z12.h, p0/z, [%[b_ptr0]]\n"
- "mov z30.h, #0\n"
- "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z31.h, #0\n"
- "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rh z15.h, p7/z, [%[betaptr]]\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "fmul z16.h, p7/m, z16.h, z15.h\n"
- "ld1h z20.h, p0/z, [c_ptr1]\n"
- "fmul z17.h, p7/m, z17.h, z15.h\n"
- "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z18.h, p7/m, z18.h, z15.h\n"
- "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z19.h, p7/m, z19.h, z15.h\n"
- "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z20.h, p7/m, z20.h, z15.h\n"
- "ld1h z24.h, p0/z, [c_ptr2]\n"
- "fmul z21.h, p7/m, z21.h, z15.h\n"
- "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z22.h, p7/m, z22.h, z15.h\n"
- "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z23.h, p7/m, z23.h, z15.h\n"
- "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z24.h, p7/m, z24.h, z15.h\n"
- "ld1h z28.h, p0/z, [c_ptr3]\n"
- "fmul z25.h, p7/m, z25.h, z15.h\n"
- "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n"
- "fmul z26.h, p7/m, z26.h, z15.h\n"
- "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n"
- "fmul z27.h, p7/m, z27.h, z15.h\n"
- "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n"
- "fmul z28.h, p7/m, z28.h, z15.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "fmul z29.h, p7/m, z29.h, z15.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "fmul z30.h, p7/m, z30.h, z15.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "fmul z31.h, p7/m, z31.h, z15.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1h z8.h, p0/z, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
"ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z23.d, z19.d\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
+ "mov z27.d, z19.d\n"
"ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z31.d, z19.d\n"
"ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.h, z8.h, z1.h[0]\n"
@@ -2959,23 +2895,23 @@
"fmla z23.h, z15.h, z5.h[7]\n"
"fmla z27.h, z15.h, z6.h[7]\n"
"fmla z31.h, z15.h, z7.h[7]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
+ "cbz %[regs], 3f\n"
+ "fmla z16.h, z8.h, z0.h[0]\n"
"ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
+ "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z28.h, z8.h, z3.h[0]\n"
+ "fmla z24.h, z8.h, z2.h[0]\n"
"ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
+ "fmla z28.h, z8.h, z3.h[0]\n"
"ld1rqh z7.h, p7/z, [a_ptr3]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.h, z9.h, z2.h[0]\n"
"fmla z29.h, z9.h, z3.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
@@ -3143,11 +3079,15 @@
"fmla z20.h, z8.h, z5.h[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z24.h, z8.h, z6.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z28.h, z8.h, z7.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"fmla z17.h, z9.h, z4.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"fmla z21.h, z9.h, z5.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z25.h, z9.h, z6.h[0]\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"fmla z29.h, z9.h, z7.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z4.h[0]\n"
@@ -3297,7 +3237,7 @@
"fmla z23.h, z15.h, z5.h[7]\n"
"fmla z27.h, z15.h, z6.h[7]\n"
"fmla z31.h, z15.h, z7.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -3320,7 +3260,7 @@
"fmla z23.h, z11.h, z1.h[0]\n"
"fmla z27.h, z11.h, z2.h[0]\n"
"fmla z31.h, z11.h, z3.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -3343,7 +3283,7 @@
"fmla z23.h, z15.h, z1.h[1]\n"
"fmla z27.h, z15.h, z2.h[1]\n"
"fmla z31.h, z15.h, z3.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -3366,7 +3306,7 @@
"fmla z23.h, z11.h, z1.h[2]\n"
"fmla z27.h, z11.h, z2.h[2]\n"
"fmla z31.h, z11.h, z3.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -3389,7 +3329,7 @@
"fmla z23.h, z15.h, z1.h[3]\n"
"fmla z27.h, z15.h, z2.h[3]\n"
"fmla z31.h, z15.h, z3.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -3412,7 +3352,7 @@
"fmla z23.h, z11.h, z1.h[4]\n"
"fmla z27.h, z11.h, z2.h[4]\n"
"fmla z31.h, z11.h, z3.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -3435,7 +3375,7 @@
"fmla z23.h, z15.h, z1.h[5]\n"
"fmla z27.h, z15.h, z2.h[5]\n"
"fmla z31.h, z15.h, z3.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -3457,27 +3397,30 @@
"fmla z23.h, z11.h, z1.h[6]\n"
"fmla z27.h, z11.h, z2.h[6]\n"
"fmla z31.h, z11.h, z3.h[6]\n"
- "b 5f\n"
- "4:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
+ "fmla z20.h, z8.h, z1.h[0]\n"
"ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z28.h, z8.h, z3.h[0]\n"
+ "fmla z24.h, z8.h, z2.h[0]\n"
"ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
+ "fmla z28.h, z8.h, z3.h[0]\n"
"ld1rqh z7.h, p6/z, [a_ptr3]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
+ "fmla z17.h, z9.h, z0.h[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
+ "fmla z21.h, z9.h, z1.h[0]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.h, z9.h, z2.h[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z29.h, z9.h, z3.h[0]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.h, z10.h, z0.h[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"fmla z22.h, z10.h, z1.h[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"fmla z26.h, z10.h, z2.h[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"fmla z30.h, z10.h, z3.h[0]\n"
"ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.h, z11.h, z0.h[0]\n"
@@ -3622,7 +3565,7 @@
"fmla z23.h, z15.h, z1.h[7]\n"
"fmla z27.h, z15.h, z2.h[7]\n"
"fmla z31.h, z15.h, z3.h[7]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -3645,7 +3588,7 @@
"fmla z23.h, z11.h, z5.h[0]\n"
"fmla z27.h, z11.h, z6.h[0]\n"
"fmla z31.h, z11.h, z7.h[0]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -3668,7 +3611,7 @@
"fmla z23.h, z15.h, z5.h[1]\n"
"fmla z27.h, z15.h, z6.h[1]\n"
"fmla z31.h, z15.h, z7.h[1]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -3691,7 +3634,7 @@
"fmla z23.h, z11.h, z5.h[2]\n"
"fmla z27.h, z11.h, z6.h[2]\n"
"fmla z31.h, z11.h, z7.h[2]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -3714,7 +3657,7 @@
"fmla z23.h, z15.h, z5.h[3]\n"
"fmla z27.h, z15.h, z6.h[3]\n"
"fmla z31.h, z15.h, z7.h[3]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
@@ -3737,7 +3680,7 @@
"fmla z23.h, z11.h, z5.h[4]\n"
"fmla z27.h, z11.h, z6.h[4]\n"
"fmla z31.h, z11.h, z7.h[4]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1h z12.h, p0/z, [%[b_ptr0]]\n"
@@ -3760,7 +3703,7 @@
"fmla z23.h, z15.h, z5.h[5]\n"
"fmla z27.h, z15.h, z6.h[5]\n"
"fmla z31.h, z15.h, z7.h[5]\n"
- "b.eq 5f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1h z8.h, p0/z, [%[b_ptr0]]\n"
"ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -3782,17 +3725,51 @@
"fmla z23.h, z11.h, z5.h[6]\n"
"fmla z27.h, z11.h, z6.h[6]\n"
"fmla z31.h, z11.h, z7.h[6]\n"
- "5:\n"
+ "4:\n"
+ "ld1rh z14.h, p7/z, [%[minptr]]\n"
+ "ld1rh z15.h, p7/z, [%[maxptr]]\n"
+ "fmax z16.h, p7/m, z16.h, z14.h\n"
+ "fmax z17.h, p7/m, z17.h, z14.h\n"
+ "fmax z18.h, p7/m, z18.h, z14.h\n"
+ "fmax z19.h, p7/m, z19.h, z14.h\n"
+ "fmin z16.h, p7/m, z16.h, z15.h\n"
+ "fmin z17.h, p7/m, z17.h, z15.h\n"
+ "fmin z18.h, p7/m, z18.h, z15.h\n"
+ "fmin z19.h, p7/m, z19.h, z15.h\n"
"st1h z16.h, p0, [%[c_ptr0]]\n"
+ "fmax z20.h, p7/m, z20.h, z14.h\n"
+ "fmax z21.h, p7/m, z21.h, z14.h\n"
+ "fmax z22.h, p7/m, z22.h, z14.h\n"
"st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.h, p7/m, z23.h, z14.h\n"
+ "fmin z20.h, p7/m, z20.h, z15.h\n"
+ "fmin z21.h, p7/m, z21.h, z15.h\n"
"st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.h, p7/m, z22.h, z15.h\n"
+ "fmin z23.h, p7/m, z23.h, z15.h\n"
+ "fmax z24.h, p7/m, z24.h, z14.h\n"
"st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.h, p7/m, z25.h, z14.h\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.h, p7/m, z26.h, z14.h\n"
"st1h z20.h, p0, [c_ptr1]\n"
+ "fmin z24.h, p7/m, z24.h, z15.h\n"
+ "fmin z25.h, p7/m, z25.h, z15.h\n"
+ "fmax z27.h, p7/m, z27.h, z14.h\n"
"st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.h, p7/m, z26.h, z15.h\n"
+ "fmax z28.h, p7/m, z28.h, z14.h\n"
+ "fmax z29.h, p7/m, z29.h, z14.h\n"
"st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n"
+ "fmin z27.h, p7/m, z27.h, z15.h\n"
+ "fmax z30.h, p7/m, z30.h, z14.h\n"
+ "fmin z28.h, p7/m, z28.h, z15.h\n"
"st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n"
+ "fmin z29.h, p7/m, z29.h, z15.h\n"
+ "fmax z31.h, p7/m, z31.h, z14.h\n"
+ "fmin z30.h, p7/m, z30.h, z15.h\n"
"st1h z24.h, p0, [c_ptr2]\n"
+ "fmin z31.h, p7/m, z31.h, z15.h\n"
"st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n"
"st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n"
"st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n"
@@ -3807,11 +3784,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp
index d7f9f20..19e5fbd 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,13 +25,11 @@
#ifdef __ARM_FEATURE_SVE
-
-
namespace arm_gemm
{
// Actual kernel implementations
-void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool);
class native_fp32_mla_4VLx4
{
@@ -39,10 +37,10 @@
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+ typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -52,20 +50,32 @@
return get_vector_length<float>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 1;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return true;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return true;
+ }
+
// Default to the generic kernel
kern_type kernel=sve_native_fp32_mla_4VLx4;
- native_fp32_mla_4VLx4(const CPUInfo *ci)
- {
-
- }
+ native_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp
index 6e22566..3fc0e5f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -25,30 +25,49 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
+void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) {
const long loops_count = ((K + 4) / 8) - 1;
K -= loops_count * 8;
const long regs_count = (K / 4) - 1;
K -= (regs_count + 1) * 4;
const long leftovers = K;
+ float nullbias[256];
+ if (!append && !bias) {
+ memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float)));
+ }
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ const float * const minptr = &minval;
+ const float * const maxptr = &maxval;
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
for (int y=0; y<M; y+=4) {
const float * const a_ptr0_base = A + (y * lda);
const unsigned long ldab = lda * sizeof(float);
float *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(float);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>()));
- const float *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
@@ -56,6 +75,8 @@
const float *a_ptr0 = a_ptr0_base;
const float *b_ptr0 = B + x0;
long ldbb = ldb * sizeof(float);
+ const unsigned long ldcb = ldc * sizeof(float);
+ const float *biasptr = bias ? bias+x0 : nullbias;
switch(M-y) {
case 1:
@@ -64,191 +85,173 @@
"whilelt p0.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"ptrue p7.s\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
"whilelt p1.s, %[temp], %[width]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
"incw %[temp], all, mul #1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
"whilelt p2.s, %[temp], %[width]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"incw %[temp], all, mul #1\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "mov z18.s, #0\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z19.s, #0\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "b 2f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "cbz %[loops], 1f\n"
"2:\n"
- "cbz %[loops], 3f\n"
- "4:\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z17.s, z9.s, z0.s[0]\n"
"ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z16.s, z12.s, z0.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z12.s, z0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z8.s, z4.s[0]\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z12.s, z4.s[3]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z4.s[3]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z4.s[3]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z15.s, z4.s[3]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "b.ne 4b\n"
- "3:\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[regs], 5f\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "cbz %[regs], 3f\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
"ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z8.s, z4.s[0]\n"
"ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z17.s, z13.s, z4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z16.s, z8.s, z4.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z12.s, z4.s[3]\n"
"fmla z17.s, z13.s, z4.s[3]\n"
"fmla z18.s, z14.s, z4.s[3]\n"
"fmla z19.s, z15.s, z4.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -259,7 +262,7 @@
"fmla z17.s, z9.s, z0.s[0]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -270,7 +273,7 @@
"fmla z17.s, z13.s, z0.s[1]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
"fmla z19.s, z15.s, z0.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -280,37 +283,37 @@
"fmla z17.s, z9.s, z0.s[2]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
"fmla z19.s, z11.s, z0.s[2]\n"
- "b 6f\n"
- "5:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
"ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"fmla z16.s, z12.s, z0.s[3]\n"
"fmla z17.s, z13.s, z0.s[3]\n"
"fmla z18.s, z14.s, z0.s[3]\n"
"fmla z19.s, z15.s, z0.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -321,7 +324,7 @@
"fmla z17.s, z9.s, z4.s[0]\n"
"fmla z18.s, z10.s, z4.s[0]\n"
"fmla z19.s, z11.s, z4.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -332,7 +335,7 @@
"fmla z17.s, z13.s, z4.s[1]\n"
"fmla z18.s, z14.s, z4.s[1]\n"
"fmla z19.s, z15.s, z4.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -342,14 +345,24 @@
"fmla z17.s, z9.s, z4.s[2]\n"
"fmla z18.s, z10.s, z4.s[2]\n"
"fmla z19.s, z11.s, z4.s[2]\n"
- "6:\n"
+ "4:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -363,61 +376,33 @@
"whilelt p0.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
"ptrue p7.s\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
"whilelt p1.s, %[temp], %[width]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
"incw %[temp], all, mul #1\n"
+ "mov z20.d, z16.d\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"whilelt p2.s, %[temp], %[width]\n"
+ "mov z21.d, z17.d\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "mov z19.s, #0\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.s, #0\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z21.s, #0\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z23.s, #0\n"
- "b 2f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "fmul z20.s, p7/m, z20.s, z15.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "fmul z21.s, p7/m, z21.s, z15.s\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmul z22.s, p7/m, z22.s, z15.s\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmul z23.s, p7/m, z23.s, z15.s\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "2:\n"
"add a_ptr1, a_ptr1, #0x10\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "mov z22.d, z18.d\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "cbz %[loops], 3f\n"
- "4:\n"
+ "mov z23.d, z19.d\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
@@ -425,200 +410,202 @@
"fmla z17.s, z9.s, z0.s[0]\n"
"ld1rqw z5.s, p7/z, [a_ptr1]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
"subs %[loops], %[loops], #0x1\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
"fmla z22.s, z10.s, z1.s[2]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z0.s[3]\n"
"fmla z21.s, z13.s, z1.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[3]\n"
"fmla z22.s, z14.s, z1.s[3]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
"ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
"ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z4.s[0]\n"
"fmla z22.s, z10.s, z5.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z4.s[0]\n"
"fmla z23.s, z11.s, z5.s[0]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z4.s[1]\n"
"fmla z22.s, z14.s, z5.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z15.s, z4.s[1]\n"
"fmla z23.s, z15.s, z5.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z4.s[2]\n"
"fmla z22.s, z10.s, z5.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z4.s[2]\n"
"fmla z23.s, z11.s, z5.s[2]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z12.s, z5.s[3]\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z4.s[3]\n"
"fmla z22.s, z14.s, z5.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
- "b.ne 4b\n"
- "3:\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[regs], 5f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
+ "cbz %[regs], 3f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
"ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
"fmla z23.s, z11.s, z1.s[0]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
"fmla z22.s, z14.s, z1.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z15.s, z0.s[1]\n"
"fmla z23.s, z15.s, z1.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
"fmla z22.s, z10.s, z1.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[2]\n"
"fmla z23.s, z11.s, z1.s[2]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[3]\n"
"fmla z22.s, z14.s, z1.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z15.s, z0.s[3]\n"
"ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
"fmla z23.s, z15.s, z1.s[3]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
"ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
"fmla z22.s, z14.s, z5.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z15.s, z4.s[1]\n"
"fmla z23.s, z15.s, z5.s[1]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
"fmla z18.s, z10.s, z4.s[2]\n"
"fmla z22.s, z10.s, z5.s[2]\n"
"fmla z19.s, z11.s, z4.s[2]\n"
@@ -631,7 +618,7 @@
"fmla z22.s, z14.s, z5.s[3]\n"
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -646,7 +633,7 @@
"fmla z22.s, z10.s, z1.s[0]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
"fmla z23.s, z11.s, z1.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -661,7 +648,7 @@
"fmla z22.s, z14.s, z1.s[1]\n"
"fmla z19.s, z15.s, z0.s[1]\n"
"fmla z23.s, z15.s, z1.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -675,40 +662,41 @@
"fmla z22.s, z10.s, z1.s[2]\n"
"fmla z19.s, z11.s, z0.s[2]\n"
"fmla z23.s, z11.s, z1.s[2]\n"
- "b 6f\n"
- "5:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
"fmla z22.s, z14.s, z1.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z15.s, z0.s[1]\n"
"fmla z23.s, z15.s, z1.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
"fmla z22.s, z10.s, z1.s[2]\n"
@@ -722,7 +710,7 @@
"fmla z22.s, z14.s, z1.s[3]\n"
"fmla z19.s, z15.s, z0.s[3]\n"
"fmla z23.s, z15.s, z1.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -737,7 +725,7 @@
"fmla z22.s, z10.s, z5.s[0]\n"
"fmla z19.s, z11.s, z4.s[0]\n"
"fmla z23.s, z11.s, z5.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -752,7 +740,7 @@
"fmla z22.s, z14.s, z5.s[1]\n"
"fmla z19.s, z15.s, z4.s[1]\n"
"fmla z23.s, z15.s, z5.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -766,10 +754,28 @@
"fmla z22.s, z10.s, z5.s[2]\n"
"fmla z19.s, z11.s, z4.s[2]\n"
"fmla z23.s, z11.s, z5.s[2]\n"
- "6:\n"
+ "4:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
+ "fmax z20.s, p7/m, z20.s, z14.s\n"
+ "fmax z21.s, p7/m, z21.s, z14.s\n"
+ "fmax z22.s, p7/m, z22.s, z14.s\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.s, p7/m, z23.s, z14.s\n"
+ "fmin z20.s, p7/m, z20.s, z15.s\n"
+ "fmin z21.s, p7/m, z21.s, z15.s\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.s, p7/m, z22.s, z15.s\n"
+ "fmin z23.s, p7/m, z23.s, z15.s\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
"st1w z20.s, p0, [c_ptr1]\n"
@@ -779,7 +785,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -791,85 +797,45 @@
"c_ptr2 .req X3\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"whilelt p6.s, %[temp], %[leftovers]\n"
"whilelt p0.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
"ptrue p7.s\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
"whilelt p1.s, %[temp], %[width]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
"incw %[temp], all, mul #1\n"
+ "mov z20.d, z16.d\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "mov z24.d, z16.d\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
"whilelt p2.s, %[temp], %[width]\n"
+ "mov z21.d, z17.d\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "mov z25.d, z17.d\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"incw %[temp], all, mul #1\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "mov z20.s, #0\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.s, #0\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z23.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z24.s, #0\n"
+ "mov z22.d, z18.d\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "mov z25.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "mov z26.s, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "mov z27.s, #0\n"
- "b 2f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "fmul z20.s, p7/m, z20.s, z15.s\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z21.s, p7/m, z21.s, z15.s\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z22.s, p7/m, z22.s, z15.s\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z23.s, p7/m, z23.s, z15.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "fmul z24.s, p7/m, z24.s, z15.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "fmul z25.s, p7/m, z25.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "fmul z26.s, p7/m, z26.s, z15.s\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmul z27.s, p7/m, z27.s, z15.s\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z26.d, z18.d\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
"add a_ptr2, a_ptr2, #0x10\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "2:\n"
+ "mov z23.d, z19.d\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "mov z27.d, z19.d\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 3f\n"
- "4:\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
@@ -879,28 +845,28 @@
"fmla z17.s, z9.s, z0.s[0]\n"
"ld1rqw z6.s, p7/z, [a_ptr2]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[loops], %[loops], #0x1\n"
"fmla z25.s, z9.s, z2.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z23.s, z11.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
"fmla z20.s, z12.s, z1.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
"fmla z21.s, z13.s, z1.s[1]\n"
"fmla z25.s, z13.s, z2.s[1]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -916,9 +882,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
@@ -933,9 +899,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z12.s, z1.s[3]\n"
"fmla z24.s, z12.s, z2.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z0.s[3]\n"
"fmla z21.s, z13.s, z1.s[3]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z13.s, z2.s[3]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[3]\n"
@@ -953,12 +919,12 @@
"fmla z20.s, z8.s, z5.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z24.s, z8.s, z6.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
"fmla z21.s, z9.s, z5.s[0]\n"
"fmla z25.s, z9.s, z6.s[0]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
"fmla z22.s, z10.s, z5.s[0]\n"
"fmla z26.s, z10.s, z6.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
@@ -970,9 +936,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z12.s, z5.s[1]\n"
"fmla z24.s, z12.s, z6.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z4.s[1]\n"
"fmla z21.s, z13.s, z5.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z13.s, z6.s[1]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z4.s[1]\n"
@@ -987,9 +953,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z4.s[2]\n"
"fmla z21.s, z9.s, z5.s[2]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z6.s[2]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z4.s[2]\n"
@@ -1004,9 +970,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z12.s, z5.s[3]\n"
"fmla z24.s, z12.s, z6.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z4.s[3]\n"
"fmla z21.s, z13.s, z5.s[3]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z13.s, z6.s[3]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z4.s[3]\n"
@@ -1016,37 +982,37 @@
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
- "b.ne 4b\n"
- "3:\n"
- "cbz %[regs], 5f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
+ "cbz %[regs], 3f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
"ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
"ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
"fmla z22.s, z10.s, z1.s[0]\n"
"fmla z26.s, z10.s, z2.s[0]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
"fmla z21.s, z13.s, z1.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z13.s, z2.s[1]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
@@ -1061,9 +1027,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[2]\n"
@@ -1078,9 +1044,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z12.s, z1.s[3]\n"
"fmla z24.s, z12.s, z2.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z0.s[3]\n"
"fmla z21.s, z13.s, z1.s[3]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z13.s, z2.s[3]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[3]\n"
@@ -1098,12 +1064,15 @@
"fmla z20.s, z8.s, z5.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z24.s, z8.s, z6.s[0]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "fmla z25.s, z9.s, z6.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z22.s, z10.s, z5.s[0]\n"
"fmla z26.s, z10.s, z6.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
@@ -1115,9 +1084,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z12.s, z5.s[1]\n"
"fmla z24.s, z12.s, z6.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z4.s[1]\n"
"fmla z21.s, z13.s, z5.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z13.s, z6.s[1]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z4.s[1]\n"
@@ -1152,7 +1121,7 @@
"fmla z19.s, z15.s, z4.s[3]\n"
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -1171,7 +1140,7 @@
"fmla z19.s, z11.s, z0.s[0]\n"
"fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -1190,7 +1159,7 @@
"fmla z19.s, z15.s, z0.s[1]\n"
"fmla z23.s, z15.s, z1.s[1]\n"
"fmla z27.s, z15.s, z2.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1208,36 +1177,38 @@
"fmla z19.s, z11.s, z0.s[2]\n"
"fmla z23.s, z11.s, z1.s[2]\n"
"fmla z27.s, z11.s, z2.s[2]\n"
- "b 6f\n"
- "5:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z5.s, p6/z, [a_ptr1]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
"ld1rqw z6.s, p6/z, [a_ptr2]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z16.s, z12.s, z0.s[1]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z13.s, z0.s[1]\n"
"fmla z21.s, z13.s, z1.s[1]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z13.s, z2.s[1]\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z14.s, z0.s[1]\n"
@@ -1272,7 +1243,7 @@
"fmla z19.s, z15.s, z0.s[3]\n"
"fmla z23.s, z15.s, z1.s[3]\n"
"fmla z27.s, z15.s, z2.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -1291,7 +1262,7 @@
"fmla z19.s, z11.s, z4.s[0]\n"
"fmla z23.s, z11.s, z5.s[0]\n"
"fmla z27.s, z11.s, z6.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -1310,7 +1281,7 @@
"fmla z19.s, z15.s, z4.s[1]\n"
"fmla z23.s, z15.s, z5.s[1]\n"
"fmla z27.s, z15.s, z6.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1328,14 +1299,40 @@
"fmla z19.s, z11.s, z4.s[2]\n"
"fmla z23.s, z11.s, z5.s[2]\n"
"fmla z27.s, z11.s, z6.s[2]\n"
- "6:\n"
+ "4:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
+ "fmax z20.s, p7/m, z20.s, z14.s\n"
+ "fmax z21.s, p7/m, z21.s, z14.s\n"
+ "fmax z22.s, p7/m, z22.s, z14.s\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.s, p7/m, z23.s, z14.s\n"
+ "fmin z20.s, p7/m, z20.s, z15.s\n"
+ "fmin z21.s, p7/m, z21.s, z15.s\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.s, p7/m, z22.s, z15.s\n"
+ "fmin z23.s, p7/m, z23.s, z15.s\n"
+ "fmax z24.s, p7/m, z24.s, z14.s\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.s, p7/m, z25.s, z14.s\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.s, p7/m, z26.s, z14.s\n"
"st1w z20.s, p0, [c_ptr1]\n"
+ "fmin z24.s, p7/m, z24.s, z15.s\n"
+ "fmin z25.s, p7/m, z25.s, z15.s\n"
+ "fmax z27.s, p7/m, z27.s, z14.s\n"
"st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.s, p7/m, z26.s, z15.s\n"
+ "fmin z27.s, p7/m, z27.s, z15.s\n"
"st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
"st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
"st1w z24.s, p0, [c_ptr2]\n"
@@ -1347,7 +1344,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -1362,103 +1359,53 @@
"c_ptr3 .req X5\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
"whilelt p6.s, %[temp], %[leftovers]\n"
"whilelt p0.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
"ptrue p7.s\n"
+ "ld1w z16.s, p0/z, [%[biasptr]]\n"
"whilelt p1.s, %[temp], %[width]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
"incw %[temp], all, mul #1\n"
+ "mov z20.d, z16.d\n"
+ "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
+ "mov z24.d, z16.d\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "mov z28.d, z16.d\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
"whilelt p2.s, %[temp], %[width]\n"
+ "mov z21.d, z17.d\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "mov z25.d, z17.d\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z29.d, z17.d\n"
+ "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"incw %[temp], all, mul #1\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z22.d, z18.d\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z26.d, z18.d\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "mov z20.s, #0\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "mov z21.s, #0\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z23.s, #0\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z24.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z25.s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "mov z26.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "mov z27.s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z28.s, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 2f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "fmul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "fmul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "fmul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "fmul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "fmul z20.s, p7/m, z20.s, z15.s\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "fmul z21.s, p7/m, z21.s, z15.s\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "fmul z22.s, p7/m, z22.s, z15.s\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "fmul z23.s, p7/m, z23.s, z15.s\n"
- "ld1w z28.s, p0/z, [c_ptr3]\n"
- "fmul z24.s, p7/m, z24.s, z15.s\n"
- "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "fmul z25.s, p7/m, z25.s, z15.s\n"
- "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "fmul z26.s, p7/m, z26.s, z15.s\n"
- "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "fmul z27.s, p7/m, z27.s, z15.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "fmul z28.s, p7/m, z28.s, z15.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "fmul z29.s, p7/m, z29.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "fmul z30.s, p7/m, z30.s, z15.s\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmul z31.s, p7/m, z31.s, z15.s\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "mov z30.d, z18.d\n"
"add a_ptr2, a_ptr2, #0x10\n"
"add a_ptr3, a_ptr3, #0x10\n"
+ "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "2:\n"
+ "mov z23.d, z19.d\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "mov z27.d, z19.d\n"
"ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z31.d, z19.d\n"
"ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 3f\n"
- "4:\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
"fmla z20.s, z8.s, z1.s[0]\n"
@@ -1470,23 +1417,23 @@
"fmla z17.s, z9.s, z0.s[0]\n"
"ld1rqw z7.s, p7/z, [a_ptr3]\n"
"fmla z21.s, z9.s, z1.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[loops], %[loops], #0x1\n"
"fmla z25.s, z9.s, z2.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z29.s, z9.s, z3.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"fmla z26.s, z10.s, z2.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"fmla z30.s, z10.s, z3.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
"fmla z23.s, z11.s, z1.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"fmla z31.s, z11.s, z3.s[0]\n"
"ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
@@ -1495,8 +1442,8 @@
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
"fmla z28.s, z12.s, z3.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
"fmla z21.s, z13.s, z1.s[1]\n"
"fmla z25.s, z13.s, z2.s[1]\n"
"fmla z29.s, z13.s, z3.s[1]\n"
@@ -1516,8 +1463,8 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z28.s, z8.s, z3.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"fmla z29.s, z9.s, z3.s[2]\n"
@@ -1537,8 +1484,8 @@
"fmla z20.s, z12.s, z1.s[3]\n"
"fmla z24.s, z12.s, z2.s[3]\n"
"fmla z28.s, z12.s, z3.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
"fmla z21.s, z13.s, z1.s[3]\n"
"fmla z25.s, z13.s, z2.s[3]\n"
"fmla z29.s, z13.s, z3.s[3]\n"
@@ -1562,9 +1509,9 @@
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z24.s, z8.s, z6.s[0]\n"
"fmla z28.s, z8.s, z7.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z17.s, z9.s, z4.s[0]\n"
"fmla z21.s, z9.s, z5.s[0]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z6.s[0]\n"
"fmla z29.s, z9.s, z7.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1583,8 +1530,8 @@
"fmla z20.s, z12.s, z5.s[1]\n"
"fmla z24.s, z12.s, z6.s[1]\n"
"fmla z28.s, z12.s, z7.s[1]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
"fmla z21.s, z13.s, z5.s[1]\n"
"fmla z25.s, z13.s, z6.s[1]\n"
"fmla z29.s, z13.s, z7.s[1]\n"
@@ -1604,8 +1551,8 @@
"fmla z20.s, z8.s, z5.s[2]\n"
"fmla z24.s, z8.s, z6.s[2]\n"
"fmla z28.s, z8.s, z7.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
"fmla z21.s, z9.s, z5.s[2]\n"
"fmla z25.s, z9.s, z6.s[2]\n"
"fmla z29.s, z9.s, z7.s[2]\n"
@@ -1625,8 +1572,8 @@
"fmla z20.s, z12.s, z5.s[3]\n"
"fmla z24.s, z12.s, z6.s[3]\n"
"fmla z28.s, z12.s, z7.s[3]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
"fmla z21.s, z13.s, z5.s[3]\n"
"fmla z25.s, z13.s, z6.s[3]\n"
"fmla z29.s, z13.s, z7.s[3]\n"
@@ -1640,28 +1587,28 @@
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
"fmla z31.s, z15.s, z7.s[3]\n"
- "b.ne 4b\n"
- "3:\n"
- "cbz %[regs], 5f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
+ "b.ne 2b\n"
+ "1:\n"
"ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
+ "cbz %[regs], 3f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
"ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z28.s, z8.s, z3.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
"ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
"ld1rqw z7.s, p7/z, [a_ptr3]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z25.s, z9.s, z2.s[0]\n"
"fmla z29.s, z9.s, z3.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z0.s[0]\n"
"fmla z22.s, z10.s, z1.s[0]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"fmla z26.s, z10.s, z2.s[0]\n"
- "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z30.s, z10.s, z3.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
@@ -1674,8 +1621,8 @@
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
"fmla z28.s, z12.s, z3.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
"fmla z21.s, z13.s, z1.s[1]\n"
"fmla z25.s, z13.s, z2.s[1]\n"
"fmla z29.s, z13.s, z3.s[1]\n"
@@ -1695,8 +1642,8 @@
"fmla z20.s, z8.s, z1.s[2]\n"
"fmla z24.s, z8.s, z2.s[2]\n"
"fmla z28.s, z8.s, z3.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
"fmla z21.s, z9.s, z1.s[2]\n"
"fmla z25.s, z9.s, z2.s[2]\n"
"fmla z29.s, z9.s, z3.s[2]\n"
@@ -1716,8 +1663,8 @@
"fmla z20.s, z12.s, z1.s[3]\n"
"fmla z24.s, z12.s, z2.s[3]\n"
"fmla z28.s, z12.s, z3.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
"fmla z21.s, z13.s, z1.s[3]\n"
"fmla z25.s, z13.s, z2.s[3]\n"
"fmla z29.s, z13.s, z3.s[3]\n"
@@ -1740,11 +1687,15 @@
"fmla z20.s, z8.s, z5.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"fmla z24.s, z8.s, z6.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"fmla z28.s, z8.s, z7.s[0]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"fmla z25.s, z9.s, z6.s[0]\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"fmla z29.s, z9.s, z7.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
"fmla z18.s, z10.s, z4.s[0]\n"
@@ -1762,8 +1713,8 @@
"fmla z20.s, z12.s, z5.s[1]\n"
"fmla z24.s, z12.s, z6.s[1]\n"
"fmla z28.s, z12.s, z7.s[1]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
"fmla z21.s, z13.s, z5.s[1]\n"
"fmla z25.s, z13.s, z6.s[1]\n"
"fmla z29.s, z13.s, z7.s[1]\n"
@@ -1810,7 +1761,7 @@
"fmla z23.s, z15.s, z5.s[3]\n"
"fmla z27.s, z15.s, z6.s[3]\n"
"fmla z31.s, z15.s, z7.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -1833,7 +1784,7 @@
"fmla z23.s, z11.s, z1.s[0]\n"
"fmla z27.s, z11.s, z2.s[0]\n"
"fmla z31.s, z11.s, z3.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -1856,7 +1807,7 @@
"fmla z23.s, z15.s, z1.s[1]\n"
"fmla z27.s, z15.s, z2.s[1]\n"
"fmla z31.s, z15.s, z3.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -1878,27 +1829,30 @@
"fmla z23.s, z11.s, z1.s[2]\n"
"fmla z27.s, z11.s, z2.s[2]\n"
"fmla z31.s, z11.s, z3.s[2]\n"
- "b 6f\n"
- "5:\n"
+ "b 4f\n"
+ "3:\n"
"fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
"ld1rqw z5.s, p6/z, [a_ptr1]\n"
- "fmla z28.s, z8.s, z3.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
"ld1rqw z6.s, p6/z, [a_ptr2]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
"ld1rqw z7.s, p6/z, [a_ptr3]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "fmla z29.s, z9.s, z3.s[0]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"fmla z30.s, z10.s, z3.s[0]\n"
"ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
"fmla z19.s, z11.s, z0.s[0]\n"
@@ -1911,8 +1865,8 @@
"fmla z20.s, z12.s, z1.s[1]\n"
"fmla z24.s, z12.s, z2.s[1]\n"
"fmla z28.s, z12.s, z3.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
"fmla z21.s, z13.s, z1.s[1]\n"
"fmla z25.s, z13.s, z2.s[1]\n"
"fmla z29.s, z13.s, z3.s[1]\n"
@@ -1959,7 +1913,7 @@
"fmla z23.s, z15.s, z1.s[3]\n"
"fmla z27.s, z15.s, z2.s[3]\n"
"fmla z31.s, z15.s, z3.s[3]\n"
- "cbz %[blocks], 6f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
@@ -1982,7 +1936,7 @@
"fmla z23.s, z11.s, z5.s[0]\n"
"fmla z27.s, z11.s, z6.s[0]\n"
"fmla z31.s, z11.s, z7.s[0]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"subs %[blocks], %[blocks], #0x1\n"
"ld1w z12.s, p0/z, [%[b_ptr0]]\n"
@@ -2005,7 +1959,7 @@
"fmla z23.s, z15.s, z5.s[1]\n"
"fmla z27.s, z15.s, z6.s[1]\n"
"fmla z31.s, z15.s, z7.s[1]\n"
- "b.eq 6f\n"
+ "b.eq 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"ld1w z8.s, p0/z, [%[b_ptr0]]\n"
"ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
@@ -2027,17 +1981,51 @@
"fmla z23.s, z11.s, z5.s[2]\n"
"fmla z27.s, z11.s, z6.s[2]\n"
"fmla z31.s, z11.s, z7.s[2]\n"
- "6:\n"
+ "4:\n"
+ "ld1rw z14.s, p7/z, [%[minptr]]\n"
+ "ld1rw z15.s, p7/z, [%[maxptr]]\n"
+ "fmax z16.s, p7/m, z16.s, z14.s\n"
+ "fmax z17.s, p7/m, z17.s, z14.s\n"
+ "fmax z18.s, p7/m, z18.s, z14.s\n"
+ "fmax z19.s, p7/m, z19.s, z14.s\n"
+ "fmin z16.s, p7/m, z16.s, z15.s\n"
+ "fmin z17.s, p7/m, z17.s, z15.s\n"
+ "fmin z18.s, p7/m, z18.s, z15.s\n"
+ "fmin z19.s, p7/m, z19.s, z15.s\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
+ "fmax z20.s, p7/m, z20.s, z14.s\n"
+ "fmax z21.s, p7/m, z21.s, z14.s\n"
+ "fmax z22.s, p7/m, z22.s, z14.s\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "fmax z23.s, p7/m, z23.s, z14.s\n"
+ "fmin z20.s, p7/m, z20.s, z15.s\n"
+ "fmin z21.s, p7/m, z21.s, z15.s\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "fmin z22.s, p7/m, z22.s, z15.s\n"
+ "fmin z23.s, p7/m, z23.s, z15.s\n"
+ "fmax z24.s, p7/m, z24.s, z14.s\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "fmax z25.s, p7/m, z25.s, z14.s\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "fmax z26.s, p7/m, z26.s, z14.s\n"
"st1w z20.s, p0, [c_ptr1]\n"
+ "fmin z24.s, p7/m, z24.s, z15.s\n"
+ "fmin z25.s, p7/m, z25.s, z15.s\n"
+ "fmax z27.s, p7/m, z27.s, z14.s\n"
"st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "fmin z26.s, p7/m, z26.s, z15.s\n"
+ "fmax z28.s, p7/m, z28.s, z14.s\n"
+ "fmax z29.s, p7/m, z29.s, z14.s\n"
"st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "fmin z27.s, p7/m, z27.s, z15.s\n"
+ "fmax z30.s, p7/m, z30.s, z14.s\n"
+ "fmin z28.s, p7/m, z28.s, z15.s\n"
"st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "fmin z29.s, p7/m, z29.s, z15.s\n"
+ "fmax z31.s, p7/m, z31.s, z14.s\n"
+ "fmin z30.s, p7/m, z30.s, z15.s\n"
"st1w z24.s, p0, [c_ptr2]\n"
+ "fmin z31.s, p7/m, z31.s, z15.s\n"
"st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
"st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
"st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
@@ -2052,11 +2040,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp
index 8b98358..1b9d131 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -27,11 +27,12 @@
#include <cstdint>
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int);
+void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
class native_s8s32_dot_4VLx4
{
@@ -39,10 +40,10 @@
typedef int8_t operand_type;
typedef int32_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int);
+ typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -52,20 +53,32 @@
return get_vector_length<int32_t>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
// Default to the generic kernel
kern_type kernel=sve_native_s8s32_dot_4VLx4;
- native_s8s32_dot_4VLx4(const CPUInfo *ci)
- {
-
- }
+ native_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp
index abee1bb..95cf88a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp
@@ -25,14 +25,14 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0);
+void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) {
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
const long regs_count = (K / 16) - 1;
@@ -46,11 +46,9 @@
const unsigned long ldab = lda * sizeof(int8_t);
int32_t *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(int32_t);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>()));
- const int32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
@@ -62,95 +60,68 @@
const int8_t *b_ptr2 = b_ptr1 + ldb;
const int8_t *b_ptr3 = b_ptr2 + ldb;
long ldbb = ldb * sizeof(int8_t) * 4;
+ const unsigned long ldcb = ldc * sizeof(int32_t);
switch(M-y) {
case 1:
__asm __volatile (
+ "mov z16.s, #0\n"
"whilelt p6.b, %[temp], %[leftovers]\n"
+ "mov z17.s, #0\n"
"whilelt p0.s, %[temp], %[width]\n"
+ "mov z18.s, #0\n"
"whilelt p4.b, %[temp], %[width]\n"
+ "mov z19.s, #0\n"
"incw %[temp], all, mul #1\n"
"ptrue p7.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"whilelt p1.s, %[temp], %[width]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"incw %[temp], all, mul #1\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mov z18.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "mov z19.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip2 z11.b, z8.b, z9.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
"zip1 z10.b, z10.b, z8.b\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "whilelt p3.s, %[temp], %[width]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z8.b, z9.b, z10.b\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "subs %[loops], %[loops], #0x1\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"sdot z16.s, z8.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"sdot z17.s, z9.b, z0.b[0]\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "zip2 z15.b, z12.b, z13.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"zip1 z13.b, z12.b, z13.b\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"zip1 z12.b, z13.b, z14.b\n"
"zip2 z13.b, z13.b, z14.b\n"
"zip1 z14.b, z15.b, z8.b\n"
@@ -289,32 +260,32 @@
"sdot z19.s, z15.b, z4.b[3]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "b.ne 3b\n"
- "2:\n"
"zip1 z10.b, z10.b, z8.b\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z9.b, z9.b, z10.b\n"
"zip1 z10.b, z11.b, z12.b\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z14.b, z15.b, z8.b\n"
@@ -377,6 +348,7 @@
"zip2 z12.b, z10.b, z8.b\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip1 z10.b, z10.b, z8.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"zip1 z8.b, z9.b, z10.b\n"
"zip2 z9.b, z9.b, z10.b\n"
"zip1 z10.b, z11.b, z12.b\n"
@@ -437,7 +409,7 @@
"sdot z17.s, z13.b, z4.b[3]\n"
"sdot z18.s, z14.b, z4.b[3]\n"
"sdot z19.s, z15.b, z4.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -459,31 +431,31 @@
"sdot z17.s, z9.b, z0.b[0]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
+ "b.eq 5f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
"b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "zip1 z12.b, z13.b, z14.b\n"
- "zip2 z13.b, z13.b, z14.b\n"
- "zip1 z14.b, z15.b, z8.b\n"
- "zip2 z15.b, z15.b, z8.b\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "b.eq 7f\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
@@ -502,31 +474,31 @@
"sdot z17.s, z9.b, z0.b[2]\n"
"sdot z18.s, z10.b, z0.b[2]\n"
"sdot z19.s, z11.b, z0.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
- "mov z13.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
+ "b 10f\n"
"9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 10f\n"
+ "8:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -540,33 +512,33 @@
"sdot z17.s, z13.b, z0.b[3]\n"
"sdot z18.s, z14.b, z0.b[3]\n"
"sdot z19.s, z15.b, z0.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
- "mov z9.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
+ "b 13f\n"
"12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 13f\n"
+ "11:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -580,33 +552,33 @@
"sdot z17.s, z9.b, z0.b[2]\n"
"sdot z18.s, z10.b, z0.b[2]\n"
"sdot z19.s, z11.b, z0.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -620,33 +592,33 @@
"sdot z17.s, z13.b, z0.b[1]\n"
"sdot z18.s, z14.b, z0.b[1]\n"
"sdot z19.s, z15.b, z0.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -660,36 +632,36 @@
"sdot z17.s, z9.b, z0.b[0]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"zip2 z13.b, z13.b, z14.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z14.b, z15.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"sdot z18.s, z14.b, z0.b[1]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
"sdot z19.s, z15.b, z0.b[1]\n"
@@ -718,7 +690,7 @@
"sdot z17.s, z13.b, z0.b[3]\n"
"sdot z18.s, z14.b, z0.b[3]\n"
"sdot z19.s, z15.b, z0.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -740,7 +712,7 @@
"sdot z17.s, z9.b, z4.b[0]\n"
"sdot z18.s, z10.b, z4.b[0]\n"
"sdot z19.s, z11.b, z4.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -762,7 +734,7 @@
"sdot z17.s, z13.b, z4.b[1]\n"
"sdot z18.s, z14.b, z4.b[1]\n"
"sdot z19.s, z15.b, z4.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -783,31 +755,31 @@
"sdot z17.s, z9.b, z4.b[2]\n"
"sdot z18.s, z10.b, z4.b[2]\n"
"sdot z19.s, z11.b, z4.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -821,33 +793,33 @@
"sdot z17.s, z13.b, z4.b[3]\n"
"sdot z18.s, z14.b, z4.b[3]\n"
"sdot z19.s, z15.b, z4.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -861,33 +833,33 @@
"sdot z17.s, z9.b, z4.b[2]\n"
"sdot z18.s, z10.b, z4.b[2]\n"
"sdot z19.s, z11.b, z4.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -901,33 +873,33 @@
"sdot z17.s, z13.b, z4.b[1]\n"
"sdot z18.s, z14.b, z4.b[1]\n"
"sdot z19.s, z15.b, z4.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -941,14 +913,14 @@
"sdot z17.s, z9.b, z4.b[0]\n"
"sdot z18.s, z10.b, z4.b[0]\n"
"sdot z19.s, z11.b, z4.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -956,125 +928,81 @@
__asm __volatile (
"a_ptr1 .req X0\n"
"c_ptr1 .req X1\n"
+ "mov z16.s, #0\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
+ "mov z17.s, #0\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "mov z18.s, #0\n"
"whilelt p6.b, %[temp], %[leftovers]\n"
+ "mov z19.s, #0\n"
"whilelt p0.s, %[temp], %[width]\n"
+ "mov z20.s, #0\n"
"whilelt p4.b, %[temp], %[width]\n"
+ "mov z21.s, #0\n"
"incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
+ "mov z22.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z23.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"whilelt p1.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "ptrue p7.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mov z19.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "mov z20.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "mov z21.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "mov z22.s, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "mov z23.s, #0\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "zip1 z8.b, z9.b, z10.b\n"
- "zip2 z9.b, z9.b, z10.b\n"
- "zip1 z10.b, z11.b, z12.b\n"
- "zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"zip2 z15.b, z12.b, z13.b\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
"zip1 z13.b, z12.b, z13.b\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
"sdot z20.s, z8.b, z1.b[0]\n"
"subs %[loops], %[loops], #0x1\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
"zip1 z14.b, z15.b, z8.b\n"
"add a_ptr1, a_ptr1, #0x20\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"sdot z22.s, z10.b, z1.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
@@ -1244,38 +1172,38 @@
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
"sdot z19.s, z15.b, z4.b[3]\n"
"sdot z23.s, z15.b, z5.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z14.b, z15.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
@@ -1357,23 +1285,25 @@
"sdot z20.s, z8.b, z5.b[0]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"zip1 z14.b, z14.b, z12.b\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"sdot z17.s, z9.b, z4.b[0]\n"
"sdot z21.s, z9.b, z5.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
"zip1 z14.b, z15.b, z8.b\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"sdot z22.s, z10.b, z5.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"sdot z19.s, z11.b, z4.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
@@ -1420,7 +1350,7 @@
"sdot z22.s, z14.b, z5.b[3]\n"
"sdot z19.s, z15.b, z4.b[3]\n"
"sdot z23.s, z15.b, z5.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -1446,35 +1376,35 @@
"sdot z22.s, z10.b, z1.b[0]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
+ "b.eq 5f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
"b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "zip1 z12.b, z13.b, z14.b\n"
- "zip2 z13.b, z13.b, z14.b\n"
- "zip1 z14.b, z15.b, z8.b\n"
- "zip2 z15.b, z15.b, z8.b\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "b.eq 7f\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
@@ -1497,31 +1427,31 @@
"sdot z22.s, z10.b, z1.b[2]\n"
"sdot z19.s, z11.b, z0.b[2]\n"
"sdot z23.s, z11.b, z1.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
- "mov z13.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
+ "b 10f\n"
"9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 10f\n"
+ "8:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1539,33 +1469,33 @@
"sdot z22.s, z14.b, z1.b[3]\n"
"sdot z19.s, z15.b, z0.b[3]\n"
"sdot z23.s, z15.b, z1.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
+ "b 13f\n"
+ "12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "12:\n"
+ "b 13f\n"
+ "11:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1583,33 +1513,33 @@
"sdot z22.s, z10.b, z1.b[2]\n"
"sdot z19.s, z11.b, z0.b[2]\n"
"sdot z23.s, z11.b, z1.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1627,33 +1557,33 @@
"sdot z22.s, z14.b, z1.b[1]\n"
"sdot z19.s, z15.b, z0.b[1]\n"
"sdot z23.s, z15.b, z1.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1671,35 +1601,36 @@
"sdot z22.s, z10.b, z1.b[0]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z14.b, z15.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
@@ -1746,7 +1677,7 @@
"sdot z22.s, z14.b, z1.b[3]\n"
"sdot z19.s, z15.b, z0.b[3]\n"
"sdot z23.s, z15.b, z1.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -1772,7 +1703,7 @@
"sdot z22.s, z10.b, z5.b[0]\n"
"sdot z19.s, z11.b, z4.b[0]\n"
"sdot z23.s, z11.b, z5.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -1798,7 +1729,7 @@
"sdot z22.s, z14.b, z5.b[1]\n"
"sdot z19.s, z15.b, z4.b[1]\n"
"sdot z23.s, z15.b, z5.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -1823,31 +1754,31 @@
"sdot z22.s, z10.b, z5.b[2]\n"
"sdot z19.s, z11.b, z4.b[2]\n"
"sdot z23.s, z11.b, z5.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1865,33 +1796,33 @@
"sdot z22.s, z14.b, z5.b[3]\n"
"sdot z19.s, z15.b, z4.b[3]\n"
"sdot z23.s, z15.b, z5.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1909,33 +1840,33 @@
"sdot z22.s, z10.b, z5.b[2]\n"
"sdot z19.s, z11.b, z4.b[2]\n"
"sdot z23.s, z11.b, z5.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1953,33 +1884,33 @@
"sdot z22.s, z14.b, z5.b[1]\n"
"sdot z19.s, z15.b, z4.b[1]\n"
"sdot z23.s, z15.b, z5.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1997,7 +1928,7 @@
"sdot z22.s, z10.b, z5.b[0]\n"
"sdot z19.s, z11.b, z4.b[0]\n"
"sdot z23.s, z11.b, z5.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
@@ -2010,7 +1941,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -2020,119 +1951,65 @@
"a_ptr2 .req X1\n"
"c_ptr1 .req X2\n"
"c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "whilelt p4.b, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
"mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
"mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"mov z19.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"mov z20.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
"mov z21.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
"mov z22.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
"mov z23.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "incw %[temp], all, mul #1\n"
"mov z24.s, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "mov z25.s, #0\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "mov z26.s, #0\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "zip1 z8.b, z9.b, z10.b\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z9.b, z9.b, z10.b\n"
- "zip1 z10.b, z11.b, z12.b\n"
- "zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z25.s, #0\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z26.s, #0\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z27.s, #0\n"
+ "whilelt p1.s, %[temp], %[width]\n"
"zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "incw %[temp], all, mul #1\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "ptrue p7.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "whilelt p2.s, %[temp], %[width]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"zip1 z10.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "incw %[temp], all, mul #1\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "whilelt p3.s, %[temp], %[width]\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"zip2 z15.b, z12.b, z13.b\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
"zip1 z13.b, z12.b, z13.b\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
"sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z6.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z8.b, z2.b[0]\n"
@@ -2361,24 +2238,23 @@
"sdot z19.s, z15.b, z4.b[3]\n"
"sdot z23.s, z15.b, z5.b[3]\n"
"sdot z27.s, z15.b, z6.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p7/z, [a_ptr2]\n"
"zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
@@ -2386,12 +2262,13 @@
"zip1 z14.b, z15.b, z8.b\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"sdot z25.s, z9.b, z2.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
"sdot z26.s, z10.b, z2.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
@@ -2491,8 +2368,11 @@
"sdot z16.s, z8.b, z4.b[0]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"sdot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"sdot z24.s, z8.b, z6.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"zip2 z8.b, z14.b, z12.b\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"zip1 z14.b, z14.b, z12.b\n"
"sdot z17.s, z9.b, z4.b[0]\n"
"sdot z21.s, z9.b, z5.b[0]\n"
@@ -2571,7 +2451,7 @@
"sdot z19.s, z15.b, z4.b[3]\n"
"sdot z23.s, z15.b, z5.b[3]\n"
"sdot z27.s, z15.b, z6.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -2601,7 +2481,7 @@
"sdot z19.s, z11.b, z0.b[0]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
"sdot z27.s, z11.b, z2.b[0]\n"
- "b.eq 6f\n"
+ "b.eq 5f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -2631,7 +2511,7 @@
"sdot z19.s, z15.b, z0.b[1]\n"
"sdot z23.s, z15.b, z1.b[1]\n"
"sdot z27.s, z15.b, z2.b[1]\n"
- "b.eq 7f\n"
+ "b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -2660,31 +2540,31 @@
"sdot z19.s, z11.b, z0.b[2]\n"
"sdot z23.s, z11.b, z1.b[2]\n"
"sdot z27.s, z11.b, z2.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
+ "b 10f\n"
+ "9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "9:\n"
+ "b 10f\n"
+ "8:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -2706,33 +2586,33 @@
"sdot z19.s, z15.b, z0.b[3]\n"
"sdot z23.s, z15.b, z1.b[3]\n"
"sdot z27.s, z15.b, z2.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
+ "b 13f\n"
+ "12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "12:\n"
+ "b 13f\n"
+ "11:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -2754,33 +2634,33 @@
"sdot z19.s, z11.b, z0.b[2]\n"
"sdot z23.s, z11.b, z1.b[2]\n"
"sdot z27.s, z11.b, z2.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -2802,33 +2682,33 @@
"sdot z19.s, z15.b, z0.b[1]\n"
"sdot z23.s, z15.b, z1.b[1]\n"
"sdot z27.s, z15.b, z2.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -2850,34 +2730,36 @@
"sdot z19.s, z11.b, z0.b[0]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
"sdot z27.s, z11.b, z2.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p6/z, [a_ptr2]\n"
"zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z14.b, z15.b, z8.b\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"sdot z25.s, z9.b, z2.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"sdot z26.s, z10.b, z2.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
@@ -2942,7 +2824,7 @@
"sdot z19.s, z15.b, z0.b[3]\n"
"sdot z23.s, z15.b, z1.b[3]\n"
"sdot z27.s, z15.b, z2.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -2972,7 +2854,7 @@
"sdot z19.s, z11.b, z4.b[0]\n"
"sdot z23.s, z11.b, z5.b[0]\n"
"sdot z27.s, z11.b, z6.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -3002,7 +2884,7 @@
"sdot z19.s, z15.b, z4.b[1]\n"
"sdot z23.s, z15.b, z5.b[1]\n"
"sdot z27.s, z15.b, z6.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -3031,31 +2913,31 @@
"sdot z19.s, z11.b, z4.b[2]\n"
"sdot z23.s, z11.b, z5.b[2]\n"
"sdot z27.s, z11.b, z6.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -3077,33 +2959,33 @@
"sdot z19.s, z15.b, z4.b[3]\n"
"sdot z23.s, z15.b, z5.b[3]\n"
"sdot z27.s, z15.b, z6.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -3125,33 +3007,33 @@
"sdot z19.s, z11.b, z4.b[2]\n"
"sdot z23.s, z11.b, z5.b[2]\n"
"sdot z27.s, z11.b, z6.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -3173,33 +3055,33 @@
"sdot z19.s, z15.b, z4.b[1]\n"
"sdot z23.s, z15.b, z5.b[1]\n"
"sdot z27.s, z15.b, z6.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -3221,7 +3103,7 @@
"sdot z19.s, z11.b, z4.b[0]\n"
"sdot z23.s, z11.b, z5.b[0]\n"
"sdot z27.s, z11.b, z6.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
@@ -3240,7 +3122,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -3253,175 +3135,109 @@
"c_ptr1 .req X3\n"
"c_ptr2 .req X4\n"
"c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "whilelt p4.b, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
"mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
"mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
"mov z19.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"mov z20.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"mov z21.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
"mov z22.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
"mov z23.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
"mov z24.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "whilelt p4.b, %[temp], %[width]\n"
"mov z25.s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "incw %[temp], all, mul #1\n"
"mov z26.s, #0\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "mov z27.s, #0\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip1 z8.b, z9.b, z10.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "zip2 z9.b, z9.b, z10.b\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip1 z10.b, z11.b, z12.b\n"
- "zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "mov z31.s, #0\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
- "ld1w z28.s, p0/z, [c_ptr3]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
- "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
- "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
- "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "mul z28.s, p7/m, z28.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z29.s, p7/m, z29.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z30.s, p7/m, z30.s, z15.s\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z31.s, p7/m, z31.s, z15.s\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z27.s, #0\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z28.s, #0\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "mov z29.s, #0\n"
+ "whilelt p1.s, %[temp], %[width]\n"
"zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr3, a_ptr3, #0x10\n"
+ "incw %[temp], all, mul #1\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "mov z30.s, #0\n"
+ "ptrue p7.b\n"
+ "mov z31.s, #0\n"
+ "whilelt p2.s, %[temp], %[width]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"zip1 z10.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "incw %[temp], all, mul #1\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "whilelt p3.s, %[temp], %[width]\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "subs %[loops], %[loops], #0x1\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip2 z13.b, z13.b, z14.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip1 z14.b, z15.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr3]\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"sdot z21.s, z9.b, z1.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"sdot z25.s, z9.b, z2.b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
"sdot z29.s, z9.b, z3.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"sdot z22.s, z10.b, z1.b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
"sdot z26.s, z10.b, z2.b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
"sdot z30.s, z10.b, z3.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"sdot z19.s, z11.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"sdot z23.s, z11.b, z1.b[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
"sdot z27.s, z11.b, z2.b[0]\n"
"sdot z31.s, z11.b, z3.b[0]\n"
"zip2 z11.b, z8.b, z9.b\n"
@@ -3649,29 +3465,29 @@
"sdot z23.s, z15.b, z5.b[3]\n"
"sdot z27.s, z15.b, z6.b[3]\n"
"sdot z31.s, z15.b, z7.b[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z28.s, z8.b, z3.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "zip2 z8.b, z14.b, z12.b\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
"ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "zip1 z14.b, z14.b, z12.b\n"
+ "zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
"zip1 z14.b, z15.b, z8.b\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -3797,9 +3613,13 @@
"sdot z16.s, z8.b, z4.b[0]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"sdot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"sdot z24.s, z8.b, z6.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"sdot z28.s, z8.b, z7.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"zip2 z8.b, z14.b, z12.b\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"zip1 z14.b, z14.b, z12.b\n"
"sdot z17.s, z9.b, z4.b[0]\n"
"sdot z21.s, z9.b, z5.b[0]\n"
@@ -3893,7 +3713,7 @@
"sdot z23.s, z15.b, z5.b[3]\n"
"sdot z27.s, z15.b, z6.b[3]\n"
"sdot z31.s, z15.b, z7.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -3927,43 +3747,43 @@
"sdot z23.s, z11.b, z1.b[0]\n"
"sdot z27.s, z11.b, z2.b[0]\n"
"sdot z31.s, z11.b, z3.b[0]\n"
+ "b.eq 5f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z28.s, z12.b, z3.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z29.s, z13.b, z3.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z30.s, z14.b, z3.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "sdot z31.s, z15.b, z3.b[1]\n"
"b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "zip1 z12.b, z13.b, z14.b\n"
- "zip2 z13.b, z13.b, z14.b\n"
- "zip1 z14.b, z15.b, z8.b\n"
- "zip2 z15.b, z15.b, z8.b\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "sdot z28.s, z12.b, z3.b[1]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "sdot z29.s, z13.b, z3.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "sdot z30.s, z14.b, z3.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "sdot z31.s, z15.b, z3.b[1]\n"
- "b.eq 7f\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
@@ -3994,31 +3814,31 @@
"sdot z23.s, z11.b, z1.b[2]\n"
"sdot z27.s, z11.b, z2.b[2]\n"
"sdot z31.s, z11.b, z3.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
- "mov z13.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
+ "b 10f\n"
"9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 10f\n"
+ "8:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4044,33 +3864,33 @@
"sdot z23.s, z15.b, z1.b[3]\n"
"sdot z27.s, z15.b, z2.b[3]\n"
"sdot z31.s, z15.b, z3.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
+ "b 13f\n"
+ "12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "12:\n"
+ "b 13f\n"
+ "11:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4096,33 +3916,33 @@
"sdot z23.s, z11.b, z1.b[2]\n"
"sdot z27.s, z11.b, z2.b[2]\n"
"sdot z31.s, z11.b, z3.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4148,33 +3968,33 @@
"sdot z23.s, z15.b, z1.b[1]\n"
"sdot z27.s, z15.b, z2.b[1]\n"
"sdot z31.s, z15.b, z3.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4200,37 +4020,40 @@
"sdot z23.s, z11.b, z1.b[0]\n"
"sdot z27.s, z11.b, z2.b[0]\n"
"sdot z31.s, z11.b, z3.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z28.s, z8.b, z3.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "zip2 z8.b, z14.b, z12.b\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
"ld1rqb z7.b, p6/z, [a_ptr3]\n"
- "zip1 z14.b, z14.b, z12.b\n"
+ "zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z14.b, z15.b, z8.b\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"sdot z21.s, z9.b, z1.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"sdot z25.s, z9.b, z2.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"sdot z29.s, z9.b, z3.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"sdot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"sdot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"sdot z26.s, z10.b, z2.b[0]\n"
"sdot z30.s, z10.b, z3.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
@@ -4309,7 +4132,7 @@
"sdot z23.s, z15.b, z1.b[3]\n"
"sdot z27.s, z15.b, z2.b[3]\n"
"sdot z31.s, z15.b, z3.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -4343,7 +4166,7 @@
"sdot z23.s, z11.b, z5.b[0]\n"
"sdot z27.s, z11.b, z6.b[0]\n"
"sdot z31.s, z11.b, z7.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -4377,7 +4200,7 @@
"sdot z23.s, z15.b, z5.b[1]\n"
"sdot z27.s, z15.b, z6.b[1]\n"
"sdot z31.s, z15.b, z7.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -4410,31 +4233,31 @@
"sdot z23.s, z11.b, z5.b[2]\n"
"sdot z27.s, z11.b, z6.b[2]\n"
"sdot z31.s, z11.b, z7.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4460,33 +4283,33 @@
"sdot z23.s, z15.b, z5.b[3]\n"
"sdot z27.s, z15.b, z6.b[3]\n"
"sdot z31.s, z15.b, z7.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4512,33 +4335,33 @@
"sdot z23.s, z11.b, z5.b[2]\n"
"sdot z27.s, z11.b, z6.b[2]\n"
"sdot z31.s, z11.b, z7.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4564,33 +4387,33 @@
"sdot z23.s, z15.b, z5.b[1]\n"
"sdot z27.s, z15.b, z6.b[1]\n"
"sdot z31.s, z15.b, z7.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4616,7 +4439,7 @@
"sdot z23.s, z11.b, z5.b[0]\n"
"sdot z27.s, z11.b, z6.b[0]\n"
"sdot z31.s, z11.b, z7.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
@@ -4641,11 +4464,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp
index bcbd3d3..33e3ac6 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,7 +32,7 @@
{
// Actual kernel implementations
-void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int);
+void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
class native_u8u32_dot_4VLx4
{
@@ -40,10 +40,10 @@
typedef uint8_t operand_type;
typedef uint32_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int);
+ typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
/* Kernel blocking parameters */
- static unsigned int out_height()
+ static constexpr unsigned int out_height()
{
return 4;
}
@@ -53,20 +53,32 @@
return get_vector_length<uint32_t>() * 4;
}
- static unsigned int k_unroll()
+ static constexpr unsigned int k_unroll()
{
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
// Default to the generic kernel
kern_type kernel=sve_native_u8u32_dot_4VLx4;
- native_u8u32_dot_4VLx4(const CPUInfo *ci)
- {
-
- }
+ native_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp
index cdcea59..994d608 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp
@@ -25,14 +25,14 @@
#include <algorithm>
+#include "arm_gemm.hpp"
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
- const long beta0 = (beta == 0u);
+void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) {
const long loops_count = ((K + 16) / 32) - 1;
K -= loops_count * 32;
const long regs_count = (K / 16) - 1;
@@ -46,11 +46,9 @@
const unsigned long ldab = lda * sizeof(uint8_t);
uint32_t *c_ptr0 = C + (y * ldc);
- const unsigned long ldcb = ldc * sizeof(uint32_t);
for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) {
const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>()));
- const uint32_t *betaptr = β
long loops = loops_count;
long regs = regs_count;
long temp = 0;
@@ -62,95 +60,68 @@
const uint8_t *b_ptr2 = b_ptr1 + ldb;
const uint8_t *b_ptr3 = b_ptr2 + ldb;
long ldbb = ldb * sizeof(uint8_t) * 4;
+ const unsigned long ldcb = ldc * sizeof(uint32_t);
switch(M-y) {
case 1:
__asm __volatile (
+ "mov z16.s, #0\n"
"whilelt p6.b, %[temp], %[leftovers]\n"
+ "mov z17.s, #0\n"
"whilelt p0.s, %[temp], %[width]\n"
+ "mov z18.s, #0\n"
"whilelt p4.b, %[temp], %[width]\n"
+ "mov z19.s, #0\n"
"incw %[temp], all, mul #1\n"
"ptrue p7.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"whilelt p1.s, %[temp], %[width]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"incw %[temp], all, mul #1\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mov z18.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "mov z19.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip2 z11.b, z8.b, z9.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
"zip1 z10.b, z10.b, z8.b\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "whilelt p3.s, %[temp], %[width]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z8.b, z9.b, z10.b\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "subs %[loops], %[loops], #0x1\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"udot z16.s, z8.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"udot z17.s, z9.b, z0.b[0]\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "zip2 z15.b, z12.b, z13.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"zip1 z13.b, z12.b, z13.b\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
"udot z18.s, z10.b, z0.b[0]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
"zip1 z12.b, z13.b, z14.b\n"
"zip2 z13.b, z13.b, z14.b\n"
"zip1 z14.b, z15.b, z8.b\n"
@@ -289,32 +260,32 @@
"udot z19.s, z15.b, z4.b[3]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "b.ne 3b\n"
- "2:\n"
"zip1 z10.b, z10.b, z8.b\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z9.b, z9.b, z10.b\n"
"zip1 z10.b, z11.b, z12.b\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z14.b, z15.b, z8.b\n"
@@ -377,6 +348,7 @@
"zip2 z12.b, z10.b, z8.b\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip1 z10.b, z10.b, z8.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"zip1 z8.b, z9.b, z10.b\n"
"zip2 z9.b, z9.b, z10.b\n"
"zip1 z10.b, z11.b, z12.b\n"
@@ -437,7 +409,7 @@
"udot z17.s, z13.b, z4.b[3]\n"
"udot z18.s, z14.b, z4.b[3]\n"
"udot z19.s, z15.b, z4.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -459,31 +431,31 @@
"udot z17.s, z9.b, z0.b[0]\n"
"udot z18.s, z10.b, z0.b[0]\n"
"udot z19.s, z11.b, z0.b[0]\n"
+ "b.eq 5f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
"b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "zip1 z12.b, z13.b, z14.b\n"
- "zip2 z13.b, z13.b, z14.b\n"
- "zip1 z14.b, z15.b, z8.b\n"
- "zip2 z15.b, z15.b, z8.b\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "b.eq 7f\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
@@ -502,31 +474,31 @@
"udot z17.s, z9.b, z0.b[2]\n"
"udot z18.s, z10.b, z0.b[2]\n"
"udot z19.s, z11.b, z0.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
- "mov z13.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
+ "b 10f\n"
"9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 10f\n"
+ "8:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -540,33 +512,33 @@
"udot z17.s, z13.b, z0.b[3]\n"
"udot z18.s, z14.b, z0.b[3]\n"
"udot z19.s, z15.b, z0.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
- "mov z9.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
+ "b 13f\n"
"12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 13f\n"
+ "11:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -580,33 +552,33 @@
"udot z17.s, z9.b, z0.b[2]\n"
"udot z18.s, z10.b, z0.b[2]\n"
"udot z19.s, z11.b, z0.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -620,33 +592,33 @@
"udot z17.s, z13.b, z0.b[1]\n"
"udot z18.s, z14.b, z0.b[1]\n"
"udot z19.s, z15.b, z0.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -660,36 +632,36 @@
"udot z17.s, z9.b, z0.b[0]\n"
"udot z18.s, z10.b, z0.b[0]\n"
"udot z19.s, z11.b, z0.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"zip2 z13.b, z13.b, z14.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip1 z14.b, z15.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"udot z18.s, z14.b, z0.b[1]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
"udot z19.s, z15.b, z0.b[1]\n"
@@ -718,7 +690,7 @@
"udot z17.s, z13.b, z0.b[3]\n"
"udot z18.s, z14.b, z0.b[3]\n"
"udot z19.s, z15.b, z0.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -740,7 +712,7 @@
"udot z17.s, z9.b, z4.b[0]\n"
"udot z18.s, z10.b, z4.b[0]\n"
"udot z19.s, z11.b, z4.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -762,7 +734,7 @@
"udot z17.s, z13.b, z4.b[1]\n"
"udot z18.s, z14.b, z4.b[1]\n"
"udot z19.s, z15.b, z4.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -783,31 +755,31 @@
"udot z17.s, z9.b, z4.b[2]\n"
"udot z18.s, z10.b, z4.b[2]\n"
"udot z19.s, z11.b, z4.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -821,33 +793,33 @@
"udot z17.s, z13.b, z4.b[3]\n"
"udot z18.s, z14.b, z4.b[3]\n"
"udot z19.s, z15.b, z4.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -861,33 +833,33 @@
"udot z17.s, z9.b, z4.b[2]\n"
"udot z18.s, z10.b, z4.b[2]\n"
"udot z19.s, z11.b, z4.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -901,33 +873,33 @@
"udot z17.s, z13.b, z4.b[1]\n"
"udot z18.s, z14.b, z4.b[1]\n"
"udot z19.s, z15.b, z4.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -941,14 +913,14 @@
"udot z17.s, z9.b, z4.b[0]\n"
"udot z18.s, z10.b, z4.b[0]\n"
"udot z19.s, z11.b, z4.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
"st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #4\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
);
break;
@@ -956,125 +928,81 @@
__asm __volatile (
"a_ptr1 .req X0\n"
"c_ptr1 .req X1\n"
+ "mov z16.s, #0\n"
"add a_ptr1, %[a_ptr0], %[lda]\n"
+ "mov z17.s, #0\n"
"add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "mov z18.s, #0\n"
"whilelt p6.b, %[temp], %[leftovers]\n"
+ "mov z19.s, #0\n"
"whilelt p0.s, %[temp], %[width]\n"
+ "mov z20.s, #0\n"
"whilelt p4.b, %[temp], %[width]\n"
+ "mov z21.s, #0\n"
"incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
+ "mov z22.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z23.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"whilelt p1.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "ptrue p7.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
"whilelt p2.s, %[temp], %[width]\n"
"incw %[temp], all, mul #1\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
"whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mov z19.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "mov z20.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "mov z21.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
"add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "mov z22.s, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "mov z23.s, #0\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "zip1 z8.b, z9.b, z10.b\n"
- "zip2 z9.b, z9.b, z10.b\n"
- "zip1 z10.b, z11.b, z12.b\n"
- "zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"zip2 z15.b, z12.b, z13.b\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
"zip1 z13.b, z12.b, z13.b\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
"udot z20.s, z8.b, z1.b[0]\n"
"subs %[loops], %[loops], #0x1\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
"add %[a_ptr0], %[a_ptr0], #0x20\n"
"zip1 z14.b, z15.b, z8.b\n"
"add a_ptr1, a_ptr1, #0x20\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"udot z22.s, z10.b, z1.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"udot z19.s, z11.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"udot z23.s, z11.b, z1.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
@@ -1244,38 +1172,38 @@
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
"udot z19.s, z15.b, z4.b[3]\n"
"udot z23.s, z15.b, z5.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z14.b, z15.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"udot z23.s, z11.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
@@ -1357,23 +1285,25 @@
"udot z20.s, z8.b, z5.b[0]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"zip1 z14.b, z14.b, z12.b\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"udot z17.s, z9.b, z4.b[0]\n"
"udot z21.s, z9.b, z5.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
"zip1 z14.b, z15.b, z8.b\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"udot z22.s, z10.b, z5.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"udot z19.s, z11.b, z4.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
@@ -1420,7 +1350,7 @@
"udot z22.s, z14.b, z5.b[3]\n"
"udot z19.s, z15.b, z4.b[3]\n"
"udot z23.s, z15.b, z5.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -1446,35 +1376,35 @@
"udot z22.s, z10.b, z1.b[0]\n"
"udot z19.s, z11.b, z0.b[0]\n"
"udot z23.s, z11.b, z1.b[0]\n"
+ "b.eq 5f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
"b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "zip1 z12.b, z13.b, z14.b\n"
- "zip2 z13.b, z13.b, z14.b\n"
- "zip1 z14.b, z15.b, z8.b\n"
- "zip2 z15.b, z15.b, z8.b\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "b.eq 7f\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
@@ -1497,31 +1427,31 @@
"udot z22.s, z10.b, z1.b[2]\n"
"udot z19.s, z11.b, z0.b[2]\n"
"udot z23.s, z11.b, z1.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
- "mov z13.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
+ "b 10f\n"
"9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 10f\n"
+ "8:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1539,33 +1469,33 @@
"udot z22.s, z14.b, z1.b[3]\n"
"udot z19.s, z15.b, z0.b[3]\n"
"udot z23.s, z15.b, z1.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
+ "b 13f\n"
+ "12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "12:\n"
+ "b 13f\n"
+ "11:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1583,33 +1513,33 @@
"udot z22.s, z10.b, z1.b[2]\n"
"udot z19.s, z11.b, z0.b[2]\n"
"udot z23.s, z11.b, z1.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1627,33 +1557,33 @@
"udot z22.s, z14.b, z1.b[1]\n"
"udot z19.s, z15.b, z0.b[1]\n"
"udot z23.s, z15.b, z1.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1671,35 +1601,36 @@
"udot z22.s, z10.b, z1.b[0]\n"
"udot z19.s, z11.b, z0.b[0]\n"
"udot z23.s, z11.b, z1.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip2 z8.b, z14.b, z12.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z14.b, z15.b, z8.b\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"udot z23.s, z11.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
@@ -1746,7 +1677,7 @@
"udot z22.s, z14.b, z1.b[3]\n"
"udot z19.s, z15.b, z0.b[3]\n"
"udot z23.s, z15.b, z1.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -1772,7 +1703,7 @@
"udot z22.s, z10.b, z5.b[0]\n"
"udot z19.s, z11.b, z4.b[0]\n"
"udot z23.s, z11.b, z5.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -1798,7 +1729,7 @@
"udot z22.s, z14.b, z5.b[1]\n"
"udot z19.s, z15.b, z4.b[1]\n"
"udot z23.s, z15.b, z5.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -1823,31 +1754,31 @@
"udot z22.s, z10.b, z5.b[2]\n"
"udot z19.s, z11.b, z4.b[2]\n"
"udot z23.s, z11.b, z5.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1865,33 +1796,33 @@
"udot z22.s, z14.b, z5.b[3]\n"
"udot z19.s, z15.b, z4.b[3]\n"
"udot z23.s, z15.b, z5.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1909,33 +1840,33 @@
"udot z22.s, z10.b, z5.b[2]\n"
"udot z19.s, z11.b, z4.b[2]\n"
"udot z23.s, z11.b, z5.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -1953,33 +1884,33 @@
"udot z22.s, z14.b, z5.b[1]\n"
"udot z19.s, z15.b, z4.b[1]\n"
"udot z23.s, z15.b, z5.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -1997,7 +1928,7 @@
"udot z22.s, z10.b, z5.b[0]\n"
"udot z19.s, z11.b, z4.b[0]\n"
"udot z23.s, z11.b, z5.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
@@ -2010,7 +1941,7 @@
".unreq a_ptr1\n"
".unreq c_ptr1\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
);
break;
@@ -2020,119 +1951,65 @@
"a_ptr2 .req X1\n"
"c_ptr1 .req X2\n"
"c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "whilelt p4.b, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
"mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
"mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"mov z19.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"mov z20.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
"mov z21.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
"mov z22.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
"mov z23.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "incw %[temp], all, mul #1\n"
"mov z24.s, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "mov z25.s, #0\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "mov z26.s, #0\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "zip1 z8.b, z9.b, z10.b\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip2 z9.b, z9.b, z10.b\n"
- "zip1 z10.b, z11.b, z12.b\n"
- "zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z25.s, #0\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z26.s, #0\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z27.s, #0\n"
+ "whilelt p1.s, %[temp], %[width]\n"
"zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "incw %[temp], all, mul #1\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "ptrue p7.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "whilelt p2.s, %[temp], %[width]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"zip1 z10.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "incw %[temp], all, mul #1\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "whilelt p3.s, %[temp], %[width]\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"zip2 z11.b, z11.b, z12.b\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
"zip2 z15.b, z12.b, z13.b\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
"zip1 z13.b, z12.b, z13.b\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
"udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z6.b, p7/z, [a_ptr2]\n"
"udot z24.s, z8.b, z2.b[0]\n"
@@ -2361,24 +2238,23 @@
"udot z19.s, z15.b, z4.b[3]\n"
"udot z23.s, z15.b, z5.b[3]\n"
"udot z27.s, z15.b, z6.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p7/z, [a_ptr2]\n"
"zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
@@ -2386,12 +2262,13 @@
"zip1 z14.b, z15.b, z8.b\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"udot z25.s, z9.b, z2.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"udot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
"udot z26.s, z10.b, z2.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"udot z19.s, z11.b, z0.b[0]\n"
@@ -2491,8 +2368,11 @@
"udot z16.s, z8.b, z4.b[0]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"udot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"udot z24.s, z8.b, z6.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"zip2 z8.b, z14.b, z12.b\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"zip1 z14.b, z14.b, z12.b\n"
"udot z17.s, z9.b, z4.b[0]\n"
"udot z21.s, z9.b, z5.b[0]\n"
@@ -2571,7 +2451,7 @@
"udot z19.s, z15.b, z4.b[3]\n"
"udot z23.s, z15.b, z5.b[3]\n"
"udot z27.s, z15.b, z6.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -2601,7 +2481,7 @@
"udot z19.s, z11.b, z0.b[0]\n"
"udot z23.s, z11.b, z1.b[0]\n"
"udot z27.s, z11.b, z2.b[0]\n"
- "b.eq 6f\n"
+ "b.eq 5f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -2631,7 +2511,7 @@
"udot z19.s, z15.b, z0.b[1]\n"
"udot z23.s, z15.b, z1.b[1]\n"
"udot z27.s, z15.b, z2.b[1]\n"
- "b.eq 7f\n"
+ "b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -2660,31 +2540,31 @@
"udot z19.s, z11.b, z0.b[2]\n"
"udot z23.s, z11.b, z1.b[2]\n"
"udot z27.s, z11.b, z2.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
+ "b 10f\n"
+ "9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "9:\n"
+ "b 10f\n"
+ "8:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -2706,33 +2586,33 @@
"udot z19.s, z15.b, z0.b[3]\n"
"udot z23.s, z15.b, z1.b[3]\n"
"udot z27.s, z15.b, z2.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
+ "b 13f\n"
+ "12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "12:\n"
+ "b 13f\n"
+ "11:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -2754,33 +2634,33 @@
"udot z19.s, z11.b, z0.b[2]\n"
"udot z23.s, z11.b, z1.b[2]\n"
"udot z27.s, z11.b, z2.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -2802,33 +2682,33 @@
"udot z19.s, z15.b, z0.b[1]\n"
"udot z23.s, z15.b, z1.b[1]\n"
"udot z27.s, z15.b, z2.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -2850,34 +2730,36 @@
"udot z19.s, z11.b, z0.b[0]\n"
"udot z23.s, z11.b, z1.b[0]\n"
"udot z27.s, z11.b, z2.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p6/z, [a_ptr2]\n"
"zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"zip2 z13.b, z13.b, z14.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z14.b, z15.b, z8.b\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"udot z25.s, z9.b, z2.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"udot z18.s, z10.b, z0.b[0]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"udot z26.s, z10.b, z2.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"udot z19.s, z11.b, z0.b[0]\n"
@@ -2942,7 +2824,7 @@
"udot z19.s, z15.b, z0.b[3]\n"
"udot z23.s, z15.b, z1.b[3]\n"
"udot z27.s, z15.b, z2.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -2972,7 +2854,7 @@
"udot z19.s, z11.b, z4.b[0]\n"
"udot z23.s, z11.b, z5.b[0]\n"
"udot z27.s, z11.b, z6.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -3002,7 +2884,7 @@
"udot z19.s, z15.b, z4.b[1]\n"
"udot z23.s, z15.b, z5.b[1]\n"
"udot z27.s, z15.b, z6.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -3031,31 +2913,31 @@
"udot z19.s, z11.b, z4.b[2]\n"
"udot z23.s, z11.b, z5.b[2]\n"
"udot z27.s, z11.b, z6.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -3077,33 +2959,33 @@
"udot z19.s, z15.b, z4.b[3]\n"
"udot z23.s, z15.b, z5.b[3]\n"
"udot z27.s, z15.b, z6.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -3125,33 +3007,33 @@
"udot z19.s, z11.b, z4.b[2]\n"
"udot z23.s, z11.b, z5.b[2]\n"
"udot z27.s, z11.b, z6.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -3173,33 +3055,33 @@
"udot z19.s, z15.b, z4.b[1]\n"
"udot z23.s, z15.b, z5.b[1]\n"
"udot z27.s, z15.b, z6.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -3221,7 +3103,7 @@
"udot z19.s, z11.b, z4.b[0]\n"
"udot z23.s, z11.b, z5.b[0]\n"
"udot z27.s, z11.b, z6.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
@@ -3240,7 +3122,7 @@
".unreq c_ptr1\n"
".unreq c_ptr2\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
);
break;
@@ -3253,175 +3135,109 @@
"c_ptr1 .req X3\n"
"c_ptr2 .req X4\n"
"c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "whilelt p4.b, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbz %[beta0], 1f\n"
"mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
"mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
"mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
"mov z19.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
"mov z20.s, #0\n"
- "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
"mov z21.s, #0\n"
- "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
"mov z22.s, #0\n"
- "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
"mov z23.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z9.b, z8.b, z9.b\n"
- "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
"mov z24.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "whilelt p4.b, %[temp], %[width]\n"
"mov z25.s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip2 z12.b, z10.b, z8.b\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip1 z10.b, z10.b, z8.b\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "incw %[temp], all, mul #1\n"
"mov z26.s, #0\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "mov z27.s, #0\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip1 z8.b, z9.b, z10.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "zip2 z9.b, z9.b, z10.b\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "zip1 z10.b, z11.b, z12.b\n"
- "zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "mov z31.s, #0\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1rw z15.s, p7/z, [%[betaptr]]\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mul z16.s, p7/m, z16.s, z15.s\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "mul z17.s, p7/m, z17.s, z15.s\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "mul z18.s, p7/m, z18.s, z15.s\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "mul z19.s, p7/m, z19.s, z15.s\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "mul z20.s, p7/m, z20.s, z15.s\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "mul z21.s, p7/m, z21.s, z15.s\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mul z22.s, p7/m, z22.s, z15.s\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mul z23.s, p7/m, z23.s, z15.s\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mul z24.s, p7/m, z24.s, z15.s\n"
- "ld1w z28.s, p0/z, [c_ptr3]\n"
- "mul z25.s, p7/m, z25.s, z15.s\n"
- "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "mul z26.s, p7/m, z26.s, z15.s\n"
- "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "mul z27.s, p7/m, z27.s, z15.s\n"
- "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "mul z28.s, p7/m, z28.s, z15.s\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mul z29.s, p7/m, z29.s, z15.s\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mul z30.s, p7/m, z30.s, z15.s\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mul z31.s, p7/m, z31.s, z15.s\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z27.s, #0\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z28.s, #0\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
+ "mov z29.s, #0\n"
+ "whilelt p1.s, %[temp], %[width]\n"
"zip2 z11.b, z8.b, z9.b\n"
- "add a_ptr3, a_ptr3, #0x10\n"
+ "incw %[temp], all, mul #1\n"
"zip1 z9.b, z8.b, z9.b\n"
"ld1b z8.b, p4/z, [%[b_ptr3]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "mov z30.s, #0\n"
+ "ptrue p7.b\n"
+ "mov z31.s, #0\n"
+ "whilelt p2.s, %[temp], %[width]\n"
"zip2 z12.b, z10.b, z8.b\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"zip1 z10.b, z10.b, z8.b\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "incw %[temp], all, mul #1\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
"zip1 z8.b, z9.b, z10.b\n"
+ "whilelt p3.s, %[temp], %[width]\n"
"zip2 z9.b, z9.b, z10.b\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
"zip1 z10.b, z11.b, z12.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
"zip2 z11.b, z11.b, z12.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "subs %[loops], %[loops], #0x1\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip2 z13.b, z13.b, z14.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip1 z14.b, z15.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr3]\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"udot z21.s, z9.b, z1.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"udot z25.s, z9.b, z2.b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
"udot z29.s, z9.b, z3.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"udot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"udot z22.s, z10.b, z1.b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
"udot z26.s, z10.b, z2.b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
"udot z30.s, z10.b, z3.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
"udot z19.s, z11.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"udot z23.s, z11.b, z1.b[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
"udot z27.s, z11.b, z2.b[0]\n"
"udot z31.s, z11.b, z3.b[0]\n"
"zip2 z11.b, z8.b, z9.b\n"
@@ -3649,29 +3465,29 @@
"udot z23.s, z15.b, z5.b[3]\n"
"udot z27.s, z15.b, z6.b[3]\n"
"udot z31.s, z15.b, z7.b[3]\n"
+ "b.ne 2b\n"
+ "1:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
"ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
+ "cbz %[regs], 3f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
"ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z28.s, z8.b, z3.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "zip2 z8.b, z14.b, z12.b\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
"ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "zip1 z14.b, z14.b, z12.b\n"
+ "zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
"zip1 z14.b, z15.b, z8.b\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -3797,9 +3613,13 @@
"udot z16.s, z8.b, z4.b[0]\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
"udot z20.s, z8.b, z5.b[0]\n"
+ "addvl %[a_ptr0], %[a_ptr0], #2\n"
"udot z24.s, z8.b, z6.b[0]\n"
+ "addvl a_ptr1, a_ptr1, #2\n"
"udot z28.s, z8.b, z7.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #2\n"
"zip2 z8.b, z14.b, z12.b\n"
+ "addvl a_ptr3, a_ptr3, #2\n"
"zip1 z14.b, z14.b, z12.b\n"
"udot z17.s, z9.b, z4.b[0]\n"
"udot z21.s, z9.b, z5.b[0]\n"
@@ -3893,7 +3713,7 @@
"udot z23.s, z15.b, z5.b[3]\n"
"udot z27.s, z15.b, z6.b[3]\n"
"udot z31.s, z15.b, z7.b[3]\n"
- "cbz %[blocks], 5f\n"
+ "cbz %[blocks], 4f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -3927,43 +3747,43 @@
"udot z23.s, z11.b, z1.b[0]\n"
"udot z27.s, z11.b, z2.b[0]\n"
"udot z31.s, z11.b, z3.b[0]\n"
+ "b.eq 5f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z28.s, z12.b, z3.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z29.s, z13.b, z3.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z30.s, z14.b, z3.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "udot z31.s, z15.b, z3.b[1]\n"
"b.eq 6f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
- "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "zip2 z15.b, z12.b, z13.b\n"
- "zip1 z13.b, z12.b, z13.b\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "zip2 z8.b, z14.b, z12.b\n"
- "zip1 z14.b, z14.b, z12.b\n"
- "zip1 z12.b, z13.b, z14.b\n"
- "zip2 z13.b, z13.b, z14.b\n"
- "zip1 z14.b, z15.b, z8.b\n"
- "zip2 z15.b, z15.b, z8.b\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "udot z28.s, z12.b, z3.b[1]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "udot z29.s, z13.b, z3.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "udot z30.s, z14.b, z3.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "udot z31.s, z15.b, z3.b[1]\n"
- "b.eq 7f\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
@@ -3994,31 +3814,31 @@
"udot z23.s, z11.b, z1.b[2]\n"
"udot z27.s, z11.b, z2.b[2]\n"
"udot z31.s, z11.b, z3.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 8f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 9f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 10f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
- "10:\n"
- "mov z13.b, #0\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 11f\n"
+ "b 10f\n"
"9:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 10f\n"
+ "8:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "11:\n"
+ "10:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4044,33 +3864,33 @@
"udot z23.s, z15.b, z1.b[3]\n"
"udot z27.s, z15.b, z2.b[3]\n"
"udot z31.s, z15.b, z3.b[3]\n"
- "b 8f\n"
- "7:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "6:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 12f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 13f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "13:\n"
+ "b 13f\n"
+ "12:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 14f\n"
- "12:\n"
+ "b 13f\n"
+ "11:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "14:\n"
+ "13:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4096,33 +3916,33 @@
"udot z23.s, z11.b, z1.b[2]\n"
"udot z27.s, z11.b, z2.b[2]\n"
"udot z31.s, z11.b, z3.b[2]\n"
- "b 8f\n"
- "6:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "5:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 15f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 16f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "16:\n"
+ "b 16f\n"
+ "15:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 17f\n"
- "15:\n"
+ "b 16f\n"
+ "14:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "17:\n"
+ "16:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4148,33 +3968,33 @@
"udot z23.s, z15.b, z1.b[1]\n"
"udot z27.s, z15.b, z2.b[1]\n"
"udot z31.s, z15.b, z3.b[1]\n"
- "b 8f\n"
- "5:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "4:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 18f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 19f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "19:\n"
+ "b 19f\n"
+ "18:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 20f\n"
- "18:\n"
+ "b 19f\n"
+ "17:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "20:\n"
+ "19:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4200,37 +4020,40 @@
"udot z23.s, z11.b, z1.b[0]\n"
"udot z27.s, z11.b, z2.b[0]\n"
"udot z31.s, z11.b, z3.b[0]\n"
- "b 8f\n"
- "4:\n"
+ "b 7f\n"
+ "3:\n"
"udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
"ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z28.s, z8.b, z3.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
"ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "zip2 z8.b, z14.b, z12.b\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
"ld1rqb z7.b, p6/z, [a_ptr3]\n"
- "zip1 z14.b, z14.b, z12.b\n"
+ "zip2 z8.b, z14.b, z12.b\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
- "zip1 z12.b, z13.b, z14.b\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
- "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
"add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "addvl %[a_ptr0], %[a_ptr0], #1\n"
"zip1 z14.b, z15.b, z8.b\n"
+ "addvl a_ptr1, a_ptr1, #1\n"
"zip2 z15.b, z15.b, z8.b\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"udot z21.s, z9.b, z1.b[0]\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"udot z25.s, z9.b, z2.b[0]\n"
+ "addvl a_ptr2, a_ptr2, #1\n"
"udot z29.s, z9.b, z3.b[0]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"udot z18.s, z10.b, z0.b[0]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"udot z22.s, z10.b, z1.b[0]\n"
+ "addvl a_ptr3, a_ptr3, #1\n"
"udot z26.s, z10.b, z2.b[0]\n"
"udot z30.s, z10.b, z3.b[0]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
@@ -4309,7 +4132,7 @@
"udot z23.s, z15.b, z1.b[3]\n"
"udot z27.s, z15.b, z2.b[3]\n"
"udot z31.s, z15.b, z3.b[3]\n"
- "cbz %[blocks], 21f\n"
+ "cbz %[blocks], 20f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -4343,7 +4166,7 @@
"udot z23.s, z11.b, z5.b[0]\n"
"udot z27.s, z11.b, z6.b[0]\n"
"udot z31.s, z11.b, z7.b[0]\n"
- "b.eq 22f\n"
+ "b.eq 21f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
@@ -4377,7 +4200,7 @@
"udot z23.s, z15.b, z5.b[1]\n"
"udot z27.s, z15.b, z6.b[1]\n"
"udot z31.s, z15.b, z7.b[1]\n"
- "b.eq 23f\n"
+ "b.eq 22f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
@@ -4410,31 +4233,31 @@
"udot z23.s, z11.b, z5.b[2]\n"
"udot z27.s, z11.b, z6.b[2]\n"
"udot z31.s, z11.b, z7.b[2]\n"
- "cbz %[odds], 8f\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 23f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 24f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 25f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "25:\n"
+ "b 25f\n"
+ "24:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 26f\n"
- "24:\n"
+ "b 25f\n"
+ "23:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "26:\n"
+ "25:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4460,33 +4283,33 @@
"udot z23.s, z15.b, z5.b[3]\n"
"udot z27.s, z15.b, z6.b[3]\n"
"udot z31.s, z15.b, z7.b[3]\n"
- "b 8f\n"
- "23:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "22:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 27f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 28f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "28:\n"
+ "b 28f\n"
+ "27:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 29f\n"
- "27:\n"
+ "b 28f\n"
+ "26:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "29:\n"
+ "28:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4512,33 +4335,33 @@
"udot z23.s, z11.b, z5.b[2]\n"
"udot z27.s, z11.b, z6.b[2]\n"
"udot z31.s, z11.b, z7.b[2]\n"
- "b 8f\n"
- "22:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "21:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 30f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 31f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z13.b, p4/z, [%[b_ptr2]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "31:\n"
+ "b 31f\n"
+ "30:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
"ld1b z14.b, p4/z, [%[b_ptr1]]\n"
- "b 32f\n"
- "30:\n"
+ "b 31f\n"
+ "29:\n"
"mov z13.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z14.b, #0\n"
"ld1b z12.b, p4/z, [%[b_ptr0]]\n"
- "32:\n"
+ "31:\n"
"zip2 z15.b, z12.b, z13.b\n"
"zip1 z13.b, z12.b, z13.b\n"
"mov z12.b, #0\n"
@@ -4564,33 +4387,33 @@
"udot z23.s, z15.b, z5.b[1]\n"
"udot z27.s, z15.b, z6.b[1]\n"
"udot z31.s, z15.b, z7.b[1]\n"
- "b 8f\n"
- "21:\n"
- "cbz %[odds], 8f\n"
+ "b 7f\n"
+ "20:\n"
+ "cbz %[odds], 7f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
"subs %[odds], %[odds], #0x1\n"
"b.eq 33f\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 34f\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr2], %[b_ptr2], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z9.b, p4/z, [%[b_ptr2]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "34:\n"
+ "b 34f\n"
+ "33:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"add %[b_ptr1], %[b_ptr1], %[ldb]\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
"ld1b z10.b, p4/z, [%[b_ptr1]]\n"
- "b 35f\n"
- "33:\n"
+ "b 34f\n"
+ "32:\n"
"mov z9.b, #0\n"
"add %[b_ptr0], %[b_ptr0], %[ldb]\n"
"mov z10.b, #0\n"
"ld1b z8.b, p4/z, [%[b_ptr0]]\n"
- "35:\n"
+ "34:\n"
"zip2 z11.b, z8.b, z9.b\n"
"zip1 z9.b, z8.b, z9.b\n"
"mov z8.b, #0\n"
@@ -4616,7 +4439,7 @@
"udot z23.s, z11.b, z5.b[0]\n"
"udot z27.s, z11.b, z6.b[0]\n"
"udot z31.s, z11.b, z7.b[0]\n"
- "8:\n"
+ "7:\n"
"st1w z16.s, p0, [%[c_ptr0]]\n"
"st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
"st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
@@ -4641,11 +4464,12 @@
".unreq c_ptr2\n"
".unreq c_ptr3\n"
: [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
+ : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb)
: "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
);
break;
}
+
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp
deleted file mode 100644
index e2cc1d1..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp
+++ /dev/null
@@ -1,4264 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_smallK_fp32_mla_1VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
-
- const long loops_count = M / 4;
- const long oddrow_count = M % 4;
- const long ldab = lda * sizeof(float);
- const long ldcb = ldc * sizeof(float);
- const long odd_depth = K % 4;
- const float *betaptr = β
- long ldbb = ldb * sizeof(float);
-
- for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) {
- const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1));
- long loops = loops_count;
- long oddrows = oddrow_count;
- long temp = 0;
- const float *b_ptr0 = B + x0;
-
- const float *a_ptr0 = A;
-
- float *c_ptr0 = C + x0;
-
- switch(K) {
- case 1:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p6/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p6/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p6/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p6/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p6/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p6/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 5:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 6:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 7:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 8:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 9:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 10:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 11:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 12:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 13:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 14:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 15:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 16:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 17:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 18:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 19:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 20:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 21:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 22:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z25.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "fmla z28.s, z25.s, z0.s[1]\n"
- "fmla z29.s, z25.s, z1.s[1]\n"
- "fmla z30.s, z25.s, z2.s[1]\n"
- "fmla z31.s, z25.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "fmla z28.s, z25.s, z1.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 23:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z25.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z26.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "fmla z28.s, z25.s, z0.s[1]\n"
- "fmla z29.s, z25.s, z1.s[1]\n"
- "fmla z30.s, z25.s, z2.s[1]\n"
- "fmla z31.s, z25.s, z3.s[1]\n"
- "fmla z28.s, z26.s, z0.s[2]\n"
- "fmla z29.s, z26.s, z1.s[2]\n"
- "fmla z30.s, z26.s, z2.s[2]\n"
- "fmla z31.s, z26.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "fmla z28.s, z25.s, z1.s[1]\n"
- "fmla z28.s, z26.s, z1.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- default:
- case 24:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z25.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z26.s, p0/z, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
- "ld1w z27.s, p0/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "fmla z28.s, z25.s, z0.s[1]\n"
- "fmla z29.s, z25.s, z1.s[1]\n"
- "fmla z30.s, z25.s, z2.s[1]\n"
- "fmla z31.s, z25.s, z3.s[1]\n"
- "fmla z28.s, z26.s, z0.s[2]\n"
- "fmla z29.s, z26.s, z1.s[2]\n"
- "fmla z30.s, z26.s, z2.s[2]\n"
- "fmla z31.s, z26.s, z3.s[2]\n"
- "fmla z28.s, z27.s, z0.s[3]\n"
- "fmla z29.s, z27.s, z1.s[3]\n"
- "fmla z30.s, z27.s, z2.s[3]\n"
- "fmla z31.s, z27.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "fmla z28.s, z25.s, z1.s[1]\n"
- "fmla z28.s, z26.s, z1.s[2]\n"
- "fmla z28.s, z27.s, z1.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp
deleted file mode 100644
index 022efdf..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_smallK_hybrid_fp32_mla_1VLx4(const float *, int, const float *, float *, int, float, int, int, int);
-
-class smallK_hybrid_fp32_mla_1VLx4
-{
-public:
- typedef float operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int);
-
- /* Kernel blocking parameters */
- static unsigned int out_height()
- {
- return 4;
- }
-
- static unsigned int out_width()
- {
- return get_vector_length<float>() * 1;
- }
-
- static unsigned int k_unroll()
- {
- return 1;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 1, 1> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=sve_smallK_hybrid_fp32_mla_1VLx4;
-
- smallK_hybrid_fp32_mla_1VLx4(const CPUInfo *ci)
- {
-
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp
deleted file mode 100644
index 3e7e713..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp
+++ /dev/null
@@ -1,4004 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_smallK_hybrid_fp32_mla_1VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) {
- const long beta0 = (beta == 0.0f);
-
- const long loops_count = M / 4;
- const long oddrow_count = M % 4;
- const long ldab = lda * sizeof(float);
- const long ldcb = ldc * sizeof(float);
- const int K_stride = K;
- const long odd_depth = K % 4;
- const float *betaptr = β
-
- for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) {
- const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1));
- long loops = loops_count;
- long oddrows = oddrow_count;
- long temp = 0;
- const float *b_ptr0 = B + (K_stride * x0);
-
- const float *a_ptr0 = A;
-
- float *c_ptr0 = C + x0;
-
- switch(K) {
- case 1:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p6/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p6/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p6/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p6/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p6/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p6/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 5:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 6:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 7:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 8:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 9:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 10:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 11:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 12:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 13:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 14:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 15:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 16:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 17:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 18:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 19:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 20:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 21:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 22:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "fmla z28.s, z25.s, z0.s[1]\n"
- "fmla z29.s, z25.s, z1.s[1]\n"
- "fmla z30.s, z25.s, z2.s[1]\n"
- "fmla z31.s, z25.s, z3.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "fmla z28.s, z25.s, z1.s[1]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 23:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "fmla z28.s, z25.s, z0.s[1]\n"
- "fmla z29.s, z25.s, z1.s[1]\n"
- "fmla z30.s, z25.s, z2.s[1]\n"
- "fmla z31.s, z25.s, z3.s[1]\n"
- "fmla z28.s, z26.s, z0.s[2]\n"
- "fmla z29.s, z26.s, z1.s[2]\n"
- "fmla z30.s, z26.s, z2.s[2]\n"
- "fmla z31.s, z26.s, z3.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "fmla z28.s, z25.s, z1.s[1]\n"
- "fmla z28.s, z26.s, z1.s[2]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- default:
- case 24:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[odd_depth]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "ptrue p7.s\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z27.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "cbz %[beta0], 3f\n"
- "mov z28.s, #0\n"
- "mov z29.s, #0\n"
- "mov z30.s, #0\n"
- "mov z31.s, #0\n"
- "b 4f\n"
- "3:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z29.s, p0/z, [c_ptr1]\n"
- "ld1w z30.s, p0/z, [c_ptr2]\n"
- "ld1w z31.s, p0/z, [c_ptr3]\n"
- "4:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z29.s, z4.s, z1.s[0]\n"
- "fmla z30.s, z4.s, z2.s[0]\n"
- "fmla z31.s, z4.s, z3.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z29.s, z5.s, z1.s[1]\n"
- "fmla z30.s, z5.s, z2.s[1]\n"
- "fmla z31.s, z5.s, z3.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z29.s, z6.s, z1.s[2]\n"
- "fmla z30.s, z6.s, z2.s[2]\n"
- "fmla z31.s, z6.s, z3.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
- "fmla z29.s, z7.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
- "fmla z30.s, z7.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z7.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
- "fmla z28.s, z8.s, z0.s[0]\n"
- "fmla z29.s, z8.s, z1.s[0]\n"
- "fmla z30.s, z8.s, z2.s[0]\n"
- "fmla z31.s, z8.s, z3.s[0]\n"
- "fmla z28.s, z9.s, z0.s[1]\n"
- "fmla z29.s, z9.s, z1.s[1]\n"
- "fmla z30.s, z9.s, z2.s[1]\n"
- "fmla z31.s, z9.s, z3.s[1]\n"
- "fmla z28.s, z10.s, z0.s[2]\n"
- "fmla z29.s, z10.s, z1.s[2]\n"
- "fmla z30.s, z10.s, z2.s[2]\n"
- "fmla z31.s, z10.s, z3.s[2]\n"
- "fmla z28.s, z11.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
- "fmla z29.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
- "fmla z30.s, z11.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
- "fmla z31.s, z11.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
- "fmla z28.s, z12.s, z0.s[0]\n"
- "fmla z29.s, z12.s, z1.s[0]\n"
- "fmla z30.s, z12.s, z2.s[0]\n"
- "fmla z31.s, z12.s, z3.s[0]\n"
- "fmla z28.s, z13.s, z0.s[1]\n"
- "fmla z29.s, z13.s, z1.s[1]\n"
- "fmla z30.s, z13.s, z2.s[1]\n"
- "fmla z31.s, z13.s, z3.s[1]\n"
- "fmla z28.s, z14.s, z0.s[2]\n"
- "fmla z29.s, z14.s, z1.s[2]\n"
- "fmla z30.s, z14.s, z2.s[2]\n"
- "fmla z31.s, z14.s, z3.s[2]\n"
- "fmla z28.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z29.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
- "fmla z30.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
- "fmla z28.s, z16.s, z0.s[0]\n"
- "fmla z29.s, z16.s, z1.s[0]\n"
- "fmla z30.s, z16.s, z2.s[0]\n"
- "fmla z31.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z0.s[1]\n"
- "fmla z29.s, z17.s, z1.s[1]\n"
- "fmla z30.s, z17.s, z2.s[1]\n"
- "fmla z31.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z0.s[2]\n"
- "fmla z29.s, z18.s, z1.s[2]\n"
- "fmla z30.s, z18.s, z2.s[2]\n"
- "fmla z31.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z29.s, z19.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
- "fmla z30.s, z19.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
- "fmla z31.s, z19.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z29.s, z20.s, z1.s[0]\n"
- "fmla z30.s, z20.s, z2.s[0]\n"
- "fmla z31.s, z20.s, z3.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z29.s, z21.s, z1.s[1]\n"
- "fmla z30.s, z21.s, z2.s[1]\n"
- "fmla z31.s, z21.s, z3.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z29.s, z22.s, z1.s[2]\n"
- "fmla z30.s, z22.s, z2.s[2]\n"
- "fmla z31.s, z22.s, z3.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
- "fmla z29.s, z23.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n"
- "fmla z30.s, z23.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n"
- "fmla z31.s, z23.s, z3.s[3]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n"
- "fmla z28.s, z24.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
- "fmla z29.s, z24.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
- "fmla z30.s, z24.s, z2.s[0]\n"
- "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
- "fmla z31.s, z24.s, z3.s[0]\n"
- "fmla z28.s, z25.s, z0.s[1]\n"
- "fmla z29.s, z25.s, z1.s[1]\n"
- "fmla z30.s, z25.s, z2.s[1]\n"
- "fmla z31.s, z25.s, z3.s[1]\n"
- "fmla z28.s, z26.s, z0.s[2]\n"
- "fmla z29.s, z26.s, z1.s[2]\n"
- "fmla z30.s, z26.s, z2.s[2]\n"
- "fmla z31.s, z26.s, z3.s[2]\n"
- "fmla z28.s, z27.s, z0.s[3]\n"
- "fmla z29.s, z27.s, z1.s[3]\n"
- "fmla z30.s, z27.s, z2.s[3]\n"
- "fmla z31.s, z27.s, z3.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
- "st1w z29.s, p0, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
- "st1w z30.s, p0, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
- "st1w z31.s, p0, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[oddrows], 5f\n"
- "6:\n"
- "cbz %[beta0], 7f\n"
- "mov z28.s, #0\n"
- "b 8f\n"
- "7:\n"
- "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
- "8:\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "subs %[oddrows], %[oddrows], #0x1\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
- "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
- "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
- "fmla z28.s, z4.s, z0.s[0]\n"
- "fmla z28.s, z5.s, z0.s[1]\n"
- "fmla z28.s, z6.s, z0.s[2]\n"
- "fmla z28.s, z7.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
- "fmla z28.s, z8.s, z1.s[0]\n"
- "fmla z28.s, z9.s, z1.s[1]\n"
- "fmla z28.s, z10.s, z1.s[2]\n"
- "fmla z28.s, z11.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], %[lda]\n"
- "fmla z28.s, z12.s, z2.s[0]\n"
- "fmla z28.s, z13.s, z2.s[1]\n"
- "fmla z28.s, z14.s, z2.s[2]\n"
- "fmla z28.s, z15.s, z2.s[3]\n"
- "fmla z28.s, z16.s, z3.s[0]\n"
- "fmla z28.s, z17.s, z3.s[1]\n"
- "fmla z28.s, z18.s, z3.s[2]\n"
- "fmla z28.s, z19.s, z3.s[3]\n"
- "fmla z28.s, z20.s, z0.s[0]\n"
- "fmla z28.s, z21.s, z0.s[1]\n"
- "fmla z28.s, z22.s, z0.s[2]\n"
- "fmla z28.s, z23.s, z0.s[3]\n"
- "fmla z28.s, z24.s, z1.s[0]\n"
- "fmla z28.s, z25.s, z1.s[1]\n"
- "fmla z28.s, z26.s, z1.s[2]\n"
- "fmla z28.s, z27.s, z1.s[3]\n"
- "st1w z28.s, p0, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
- "b.ne 6b\n"
- "5:\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
- : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp
index ca90ee1..fc18cbd 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp
@@ -31,7 +31,7 @@
{
// Actual kernel implementations
-void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
class smallK_hybrid_s8s32_dot_1VLx8
{
@@ -39,7 +39,7 @@
typedef int8_t operand_type;
typedef int32_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int);
+ typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -57,15 +57,27 @@
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {};
// Default to the generic kernel
kern_type kernel=sve_smallK_hybrid_s8s32_dot_1VLx8;
- smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci)
- {
-
- }
+ smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp
index 04605b2..d9813a5 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp
@@ -25,13 +25,15 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
+void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)get_vector_length<int32_t>()) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp
index 4533014..51d3e73 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp
@@ -31,7 +31,7 @@
{
// Actual kernel implementations
-void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
class smallK_hybrid_u8u32_dot_1VLx8
{
@@ -39,7 +39,7 @@
typedef uint8_t operand_type;
typedef uint32_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int);
+ typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -57,15 +57,27 @@
return 4;
}
+ static constexpr bool supports_append()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_bias()
+ {
+ return false;
+ }
+
+ static constexpr bool supports_activation()
+ {
+ return false;
+ }
+
StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {};
// Default to the generic kernel
kern_type kernel=sve_smallK_hybrid_u8u32_dot_1VLx8;
- smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci)
- {
-
- }
+ smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp
index 7c965d3..6eed457 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp
@@ -25,13 +25,15 @@
#include <algorithm>
+#include "arm_gemm.hpp"
+
#include <cstdint>
#include "../../asmlib.hpp"
#include "../../utils.hpp"
namespace arm_gemm {
-void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
+void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)get_vector_length<uint32_t>()) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
diff --git a/src/core/NEON/kernels/arm_gemm/mergeresults.cpp b/src/core/NEON/kernels/arm_gemm/mergeresults.cpp
new file mode 100644
index 0000000..83d6bcc
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/mergeresults.cpp
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2017-2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* As some of the merges need these headers, but are all included in the
+ * arm_gemm namespace, put these headers here. */
+#include <algorithm>
+
+#include <arm_neon.h>
+
+#include "arm_gemm.hpp"
+#include "asmlib.hpp"
+#include "utils.hpp"
+
+namespace arm_gemm {
+
+template<unsigned int twidth, unsigned int height, bool sve=false, typename Tin, typename Tout>
+void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append) {
+ // For SVE cases, multiply the width up by the vector length.
+ // Use the *input* type to determine this, since this will be what the kernel operated on.
+ const int width = twidth * (sve ? get_vector_length<Tin>() : 1);
+
+ const int full_y_blocks = (ymax - y0) / height;
+ const int y_remainder = (ymax - y0) % height;
+ const int y_blocks = full_y_blocks + (y_remainder ? 1 : 0);
+
+ const int full_x_blocks = (xmax - x0) / width;
+ const int x_remainder = (xmax - x0) % width;
+ const int x_blocks = full_x_blocks + (x_remainder ? 1 : 0);
+
+ for (int y_block = 0; y_block < y_blocks; y_block++) {
+ int ybase = y0 + (y_block * height);
+
+ int fill_rows = (y_block < full_y_blocks) ? height : y_remainder;
+
+ for (int x_block = 0; x_block < x_blocks; x_block++) {
+ int xbase = x0 + (x_block * width);
+
+ int fill_cols = (x_block < full_x_blocks) ? width : x_remainder;
+
+ for (int row=0; row < fill_rows; row++) {
+ for (int col=0; col < fill_cols; col++) {
+ Tout &r = out[(ybase + row) * ldc + xbase + col];
+ Tout v = in[row * width + col];
+
+ if (append) {
+ v += r;
+ }
+
+ if (bias) {
+ v += bias[xbase + col];
+ }
+
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+
+ case Activation::Type::ReLU:
+ v = std::max(v, static_cast<Tout>(0));
+ break;
+
+ case Activation::Type::BoundedReLU:
+ v = std::max(std::min(v, static_cast<Tout>(act.param1)), static_cast<Tout>(0));
+ break;
+ }
+
+ r = v;
+ }
+ }
+
+ in += (width * height);
+ }
+ }
+}
+
+#include "merges/list.hpp"
+
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS)
+template void MergeResults<12u, 8u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool);
+#endif
+
+#if defined(__arm__) && defined(__ARM_FP16_ARGS)
+template void MergeResults<8u, 6u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool);
+#endif
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/mergeresults.hpp b/src/core/NEON/kernels/arm_gemm/mergeresults.hpp
index 04d1343..1aedd23 100644
--- a/src/core/NEON/kernels/arm_gemm/mergeresults.hpp
+++ b/src/core/NEON/kernels/arm_gemm/mergeresults.hpp
@@ -23,58 +23,9 @@
*/
#pragma once
-/* As some of the merges need these headers, but are all included in the
- * arm_gemm namespace, put these headers here. */
-#include <arm_neon.h>
-
-#include "asmlib.hpp"
-#include "utils.hpp"
-
namespace arm_gemm {
template<unsigned int twidth, unsigned int height, bool sve=false, typename Tin, typename Tout>
-inline void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout alpha, const Tout beta) {
- // For SVE cases, multiply the width up by the vector length.
- // Use the *input* type to determine this, since this will be what the kernel operated on.
- const int width = twidth * (sve ? get_vector_length<Tin>() : 1);
-
- const int full_y_blocks = (ymax - y0) / height;
- const int y_remainder = (ymax - y0) % height;
- const int y_blocks = full_y_blocks + (y_remainder ? 1 : 0);
-
- const int full_x_blocks = (xmax - x0) / width;
- const int x_remainder = (xmax - x0) % width;
- const int x_blocks = full_x_blocks + (x_remainder ? 1 : 0);
-
- for (int y_block = 0; y_block < y_blocks; y_block++) {
- int ybase = y0 + (y_block * height);
-
- int fill_rows = (y_block < full_y_blocks) ? height : y_remainder;
-
- for (int x_block = 0; x_block < x_blocks; x_block++) {
- int xbase = x0 + (x_block * width);
-
- int fill_cols = (x_block < full_x_blocks) ? width : x_remainder;
-
- for (int row=0; row < fill_rows; row++) {
- for (int col=0; col < fill_cols; col++) {
- Tout &p = out[(ybase + row) * ldc + xbase + col];
-
- // Special case for beta==0 - don't read the input;
- // (0 * x == 0) is not always true for FP types.
- if (beta == static_cast<Tout>(0)) {
- p = (alpha * in[row * width + col]);
- } else {
- p = (p * beta) + (alpha * in[row * width + col]);
- }
- }
- }
-
- in += (width * height);
- }
- }
-}
-
-#include "merges/list.hpp"
+void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp b/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp
index e1af2d4..9409646 100644
--- a/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp
@@ -28,13 +28,35 @@
#include <arm_neon.h>
template<>
-inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) {
+void MergeResults<8, 6, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) {
const float *inptr = in;
prefetch_6x(inptr);
prefetch_6x(inptr + 96);
- float32x4_t av = vdupq_n_f32(alpha);
- float32x4_t bv = vdupq_n_f32(beta);
+ float nullbias[8];
+ float minval = - std::numeric_limits<float>::infinity();
+ float maxval = std::numeric_limits<float>::infinity();
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ float32x4_t minv = vdupq_n_f32(minval);
+ float32x4_t maxv = vdupq_n_f32(maxval);
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (8 * sizeof(float)));
+ }
for (int y=y0; y<ymax; y+=8) {
float *outptr0 = out + (y * ldout) + x0;
@@ -61,16 +83,12 @@
switch ((y + 5) - ymax) {
case 4:
outptr1 = dummyres;
- // fall through
case 3:
outptr2 = dummyres;
- // fall through
case 2:
outptr3 = dummyres;
- // fall through
case 1:
outptr4 = dummyres;
- // fall through
case 0:
outptr5 = dummyres;
break;
@@ -80,24 +98,24 @@
}
}
- if (beta == 0.0f) {
- /* If beta=0, don't read the original input at all. */
+ if (append) {
+ /* Append mode: Read, activate, write. */
/* For ragged X, manually copy over the valid results. */
if ((i+7) >= xmax) {
for (int xi=0; xi<8; xi++) {
if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 8]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 8] + *outptr1), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 16]);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 16] + *outptr2), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 24]);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 24] + *outptr3), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 32]);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 32] + *outptr4), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 40]);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 40] + *outptr5), maxval);
outptr5++;
}
}
@@ -107,69 +125,100 @@
__asm __volatile (
// Rows 0-1
"VLD1.32 {d0-d3}, [%[inptr]]!\n"
+ "VLD1.32 {d8-d11}, [%[outptr0]]\n"
"VLD1.32 {d4-d7}, [%[inptr]]!\n"
+ "VLD1.32 {d12-d15}, [%[outptr1]]\n"
- "VMUL.f32 q4, q0, %q[av]\n"
+ "VADD.f32 q4, q4, q0\n"
ASM_PREFETCH("[%[inptr], #352]")
- "VMUL.f32 q5, q1, %q[av]\n"
- "VST1.32 {d8-d11}, [%[outptr0]]!\n"
+ "VADD.f32 q5, q5, q1\n"
+ "VADD.f32 q6, q6, q2\n"
+ "VADD.f32 q7, q7, q3\n"
ASM_PREFETCH("[%[inptr], #416]")
- "VMUL.f32 q6, q2, %q[av]\n"
+ "VMAX.f32 q4, q4, %q[minv]\n"
+ "VMAX.f32 q5, q5, %q[minv]\n"
+ "VMAX.f32 q6, q6, %q[minv]\n"
ASM_PREFETCH("[%[inptr], #480]")
- "VMUL.f32 q7, q3, %q[av]\n"
+ "VMAX.f32 q7, q7, %q[minv]\n"
+ "VMIN.f32 q4, q4, %q[maxv]\n"
+ "VMIN.f32 q5, q5, %q[maxv]\n"
+ "VST1.32 {d8-d11}, [%[outptr0]]!\n"
+ "VMIN.f32 q6, q6, %q[maxv]\n"
+ "VMIN.f32 q7, q7, %q[maxv]\n"
"VST1.32 {d12-d15}, [%[outptr1]]!\n"
// Rows 2-3
"VLD1.32 {d0-d3}, [%[inptr]]!\n"
+ "VLD1.32 {d8-d11}, [%[outptr2]]\n"
"VLD1.32 {d4-d7}, [%[inptr]]!\n"
+ "VLD1.32 {d12-d15}, [%[outptr3]]\n"
- "VMUL.f32 q4, q0, %q[av]\n"
+ "VADD.f32 q4, q4, q0\n"
ASM_PREFETCH("[%[outptr0], #96]")
- "VMUL.f32 q5, q1, %q[av]\n"
- "VST1.32 {d8-d11}, [%[outptr2]]!\n"
+ "VADD.f32 q5, q5, q1\n"
+ "VADD.f32 q6, q6, q2\n"
+ "VADD.f32 q7, q7, q3\n"
ASM_PREFETCH("[%[outptr1], #96]")
- "VMUL.f32 q6, q2, %q[av]\n"
- ASM_PREFETCH("[%[outptr2], #96]")
- "VMUL.f32 q7, q3, %q[av]\n"
+ "VMAX.f32 q4, q4, %q[minv]\n"
+ "VMAX.f32 q5, q5, %q[minv]\n"
+ "VMAX.f32 q6, q6, %q[minv]\n"
+ ASM_PREFETCH("[%[outptr2], #128]")
+ "VMAX.f32 q7, q7, %q[minv]\n"
+ "VMIN.f32 q4, q4, %q[maxv]\n"
+ "VMIN.f32 q5, q5, %q[maxv]\n"
+ "VST1.32 {d8-d11}, [%[outptr2]]!\n"
+ "VMIN.f32 q6, q6, %q[maxv]\n"
+ "VMIN.f32 q7, q7, %q[maxv]\n"
"VST1.32 {d12-d15}, [%[outptr3]]!\n"
// Rows 4-5
"VLD1.32 {d0-d3}, [%[inptr]]!\n"
+ "VLD1.32 {d8-d11}, [%[outptr4]]\n"
"VLD1.32 {d4-d7}, [%[inptr]]!\n"
+ "VLD1.32 {d12-d15}, [%[outptr5]]\n"
- "VMUL.f32 q4, q0, %q[av]\n"
+ "VADD.f32 q4, q4, q0\n"
ASM_PREFETCH("[%[outptr3], #96]")
- "VMUL.f32 q5, q1, %q[av]\n"
- "VST1.32 {d8-d11}, [%[outptr4]]!\n"
- ASM_PREFETCH("[%[outptr4], #96]")
- "VMUL.f32 q6, q2, %q[av]\n"
+ "VADD.f32 q5, q5, q1\n"
+ "VADD.f32 q6, q6, q2\n"
+ "VADD.f32 q7, q7, q3\n"
+ ASM_PREFETCH("[%[outptr4], #128]")
+ "VMAX.f32 q4, q4, %q[minv]\n"
+ "VMAX.f32 q5, q5, %q[minv]\n"
+ "VMAX.f32 q6, q6, %q[minv]\n"
ASM_PREFETCH("[%[outptr5], #128]")
- "VMUL.f32 q7, q3, %q[av]\n"
+ "VMAX.f32 q7, q7, %q[minv]\n"
+ "VMIN.f32 q4, q4, %q[maxv]\n"
+ "VMIN.f32 q5, q5, %q[maxv]\n"
+ "VST1.32 {d8-d11}, [%[outptr4]]!\n"
+ "VMIN.f32 q6, q6, %q[maxv]\n"
+ "VMIN.f32 q7, q7, %q[maxv]\n"
"VST1.32 {d12-d15}, [%[outptr5]]!\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
[outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [inptr] "+r" (inptr)
- : [av] "w" (av), [bv] "w" (bv)
- : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7"
+ : [minv] "w" (minv), [maxv] "w" (maxv)
+ : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "memory"
);
}
} else {
- /* Non-zero beta: Read output and apply beta. */
+ /* Bias mode: Add bias to everything, then min/max/write as before. */
+ const float *biasptr = bias ? bias + i : nullbias;
/* For ragged X, manually copy over the valid results. */
if ((i+7) >= xmax) {
- for (int xi=0; xi<8; xi++) {
+ for (int xi=0; xi<7; xi++) {
if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 8]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 8] + biasptr[xi]), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 16]) + (*outptr2 * beta);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 16] + biasptr[xi]), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 24]) + (*outptr3 * beta);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 32]) + (*outptr4 * beta);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 32] + biasptr[xi]), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 40]) + (*outptr5 * beta);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 40] + biasptr[xi]), maxval);
outptr5++;
}
}
@@ -178,68 +227,75 @@
/* Optimized routine to copy an entire block */
__asm __volatile (
// Rows 0-1
- "VLD1.32 {d8-d11}, [%[outptr0]]\n"
- "VMUL.f32 q4, q4, %q[bv]\n"
- "VLD1.32 {d12-d15}, [%[outptr1]]\n"
- "VMUL.f32 q5, q5, %q[bv]\n"
- "VLD1.32 {d0-d3}, [%[inptr]]!\n"
- "VMUL.f32 q6, q6, %q[bv]\n"
- "VLD1.32 {d4-d7}, [%[inptr]]!\n"
- "VMUL.f32 q7, q7, %q[bv]\n"
+ "VLD1.32 {d8-d11}, [%[inptr]]!\n"
+ "VLD1.32 {d0-d3}, [%[biasptr]]\n"
+ "VLD1.32 {d12-d15}, [%[inptr]]!\n"
- "VMLA.f32 q4, q0, %q[av]\n"
+ "VADD.f32 q4, q4, q0\n"
ASM_PREFETCH("[%[inptr], #352]")
- "VMLA.f32 q5, q1, %q[av]\n"
- "VST1.32 {d8-d11}, [%[outptr0]]!\n"
+ "VADD.f32 q5, q5, q1\n"
+ "VADD.f32 q6, q6, q0\n"
+ "VADD.f32 q7, q7, q1\n"
ASM_PREFETCH("[%[inptr], #416]")
- "VMLA.f32 q6, q2, %q[av]\n"
+ "VMAX.f32 q4, q4, %q[minv]\n"
+ "VMAX.f32 q5, q5, %q[minv]\n"
+ "VMAX.f32 q6, q6, %q[minv]\n"
ASM_PREFETCH("[%[inptr], #480]")
- "VMLA.f32 q7, q3, %q[av]\n"
+ "VMAX.f32 q7, q7, %q[minv]\n"
+ "VMIN.f32 q4, q4, %q[maxv]\n"
+ "VMIN.f32 q5, q5, %q[maxv]\n"
+ "VST1.32 {d8-d11}, [%[outptr0]]!\n"
+ "VMIN.f32 q6, q6, %q[maxv]\n"
+ "VMIN.f32 q7, q7, %q[maxv]\n"
"VST1.32 {d12-d15}, [%[outptr1]]!\n"
// Rows 2-3
- "VLD1.32 {d8-d11}, [%[outptr2]]\n"
- "VMUL.f32 q4, q4, %q[bv]\n"
- "VLD1.32 {d12-d15}, [%[outptr3]]\n"
- "VMUL.f32 q5, q5, %q[bv]\n"
- "VLD1.32 {d0-d3}, [%[inptr]]!\n"
- "VMUL.f32 q6, q6, %q[bv]\n"
- "VLD1.32 {d4-d7}, [%[inptr]]!\n"
- "VMUL.f32 q7, q7, %q[bv]\n"
+ "VLD1.32 {d8-d11}, [%[inptr]]!\n"
+ "VLD1.32 {d12-d15}, [%[inptr]]!\n"
- "VMLA.f32 q4, q0, %q[av]\n"
+ "VADD.f32 q4, q4, q0\n"
ASM_PREFETCH("[%[outptr0], #96]")
- "VMLA.f32 q5, q1, %q[av]\n"
- "VST1.32 {d8-d11}, [%[outptr2]]!\n"
+ "VADD.f32 q5, q5, q1\n"
+ "VADD.f32 q6, q6, q0\n"
+ "VADD.f32 q7, q7, q1\n"
ASM_PREFETCH("[%[outptr1], #96]")
- "VMLA.f32 q6, q2, %q[av]\n"
- ASM_PREFETCH("[%[outptr2], #96]")
- "VMLA.f32 q7, q3, %q[av]\n"
+ "VMAX.f32 q4, q4, %q[minv]\n"
+ "VMAX.f32 q5, q5, %q[minv]\n"
+ "VMAX.f32 q6, q6, %q[minv]\n"
+ ASM_PREFETCH("[%[outptr2], #128]")
+ "VMAX.f32 q7, q7, %q[minv]\n"
+ "VMIN.f32 q4, q4, %q[maxv]\n"
+ "VMIN.f32 q5, q5, %q[maxv]\n"
+ "VST1.32 {d8-d11}, [%[outptr2]]!\n"
+ "VMIN.f32 q6, q6, %q[maxv]\n"
+ "VMIN.f32 q7, q7, %q[maxv]\n"
"VST1.32 {d12-d15}, [%[outptr3]]!\n"
// Rows 4-5
- "VLD1.32 {d8-d11}, [%[outptr4]]\n"
- "VMUL.f32 q4, q4, %q[bv]\n"
- "VLD1.32 {d12-d15}, [%[outptr5]]\n"
- "VMUL.f32 q5, q5, %q[bv]\n"
- "VLD1.32 {d0-d3}, [%[inptr]]!\n"
- "VMUL.f32 q6, q6, %q[bv]\n"
- "VLD1.32 {d4-d7}, [%[inptr]]!\n"
- "VMUL.f32 q7, q7, %q[bv]\n"
+ "VLD1.32 {d8-d11}, [%[inptr]]!\n"
+ "VLD1.32 {d12-d15}, [%[inptr]]!\n"
- "VMLA.f32 q4, q0, %q[av]\n"
+ "VADD.f32 q4, q4, q0\n"
ASM_PREFETCH("[%[outptr3], #96]")
- "VMLA.f32 q5, q1, %q[av]\n"
- "VST1.32 {d8-d11}, [%[outptr4]]!\n"
- ASM_PREFETCH("[%[outptr4], #96]")
- "VMLA.f32 q6, q2, %q[av]\n"
+ "VADD.f32 q5, q5, q1\n"
+ "VADD.f32 q6, q6, q0\n"
+ "VADD.f32 q7, q7, q1\n"
+ ASM_PREFETCH("[%[outptr4], #128]")
+ "VMAX.f32 q4, q4, %q[minv]\n"
+ "VMAX.f32 q5, q5, %q[minv]\n"
+ "VMAX.f32 q6, q6, %q[minv]\n"
ASM_PREFETCH("[%[outptr5], #128]")
- "VMLA.f32 q7, q3, %q[av]\n"
+ "VMAX.f32 q7, q7, %q[minv]\n"
+ "VMIN.f32 q4, q4, %q[maxv]\n"
+ "VMIN.f32 q5, q5, %q[maxv]\n"
+ "VST1.32 {d8-d11}, [%[outptr4]]!\n"
+ "VMIN.f32 q6, q6, %q[maxv]\n"
+ "VMIN.f32 q7, q7, %q[maxv]\n"
"VST1.32 {d12-d15}, [%[outptr5]]!\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
[outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [inptr] "+r" (inptr)
- : [av] "w" (av), [bv] "w" (bv)
- : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7"
+ : [minv] "w" (minv), [maxv] "w" (maxv), [biasptr] "r" (biasptr)
+ : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "memory"
);
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp
deleted file mode 100644
index 9fca4e3..0000000
--- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Copyright (c) 2017-2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-template<>
-inline void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) {
- const float *inptr = in;
- prefetch_6x(inptr);
- prefetch_6x(inptr + 96);
-
- float32x4_t av = vdupq_n_f32(alpha);
- float32x4_t bv = vdupq_n_f32(beta);
-
- for (int y=y0; y<ymax; y+=8) {
- float *outptr0 = out + (y * ldout) + x0;
- float *outptr1 = outptr0 + ldout;
- float *outptr2 = outptr1 + ldout;
- float *outptr3 = outptr2 + ldout;
- float *outptr4 = outptr3 + ldout;
- float *outptr5 = outptr4 + ldout;
- float *outptr6 = outptr5 + ldout;
- float *outptr7 = outptr6 + ldout;
-
- prefetch_2x(outptr0);
- prefetch_2x(outptr1);
- prefetch_2x(outptr2);
- prefetch_2x(outptr3);
- prefetch_2x(outptr4);
- prefetch_2x(outptr5);
- prefetch_2x(outptr6);
- prefetch_2x(outptr7);
-
- for (int i=x0; i<xmax; i+=12) {
- float dummyres[12];
-
- /* Make sure we throw away results if Y isn't a multiple of 8.
- * We do this by pointing the result pointer at a dummy buffer
- * we later discard. */
- if ((y+7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- outptr1 = dummyres;
- // fall through
- case 5:
- outptr2 = dummyres;
- // fall through
- case 4:
- outptr3 = dummyres;
- // fall through
- case 3:
- outptr4 = dummyres;
- // fall through
- case 2:
- outptr5 = dummyres;
- // fall through
- case 1:
- outptr6 = dummyres;
- // fall through
- case 0:
- outptr7 = dummyres;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (beta==0.0f) {
- /* If beta==0, don't read the original input at all. */
-
- /* For ragged X, manually copy over the valid results. */
- if ((i+11) >= xmax) {
- for (int xi=0; xi<12; xi++) {
- if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]);
- outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
- outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
- outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]);
- outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]);
- outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]);
- outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]);
- outptr6++;
- *outptr7 = (alpha * inptr[xi + 84]);
- outptr7++;
- }
- }
- inptr += 96;
- } else {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- // Rows 0-1
- "LDP q0, q1, [%[inptr]]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- "LDP q2, q3, [%[inptr], #32]\n"
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- "LDP q4, q5, [%[inptr], #64]\n"
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr0]], #32\n"
- ASM_PREFETCH("[%[inptr], #768]")
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "STR q18, [%[outptr0]], #16\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr1]], #32\n"
- ASM_PREFETCH("[%[inptr], #832]")
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr1]], #16\n"
-
- // Rows 2-3
- "LDP q0, q1, [%[inptr], #96]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- "LDP q2, q3, [%[inptr], #128]\n"
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- "LDP q4, q5, [%[inptr], #160]\n"
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr2]], #32\n"
- ASM_PREFETCH("[%[inptr], #896]")
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "STR q18, [%[outptr2]], #16\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr3]], #32\n"
- ASM_PREFETCH("[%[inptr], #1024]")
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr3]], #16\n"
-
- // Rows 4-5
- "LDP q0, q1, [%[inptr], #192]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- "LDP q2, q3, [%[inptr], #224]\n"
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- "LDP q4, q5, [%[inptr], #256]\n"
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr4]], #32\n"
- ASM_PREFETCH("[%[inptr], #960]")
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "STR q18, [%[outptr4]], #16\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr5]], #32\n"
- ASM_PREFETCH("[%[inptr], #1088]")
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr5]], #16\n"
-
- // Rows 6-7
- "LDP q0, q1, [%[inptr], #288]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- "LDP q2, q3, [%[inptr], #320]\n"
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- "LDP q4, q5, [%[inptr], #352]\n"
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr6]], #32\n"
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "STR q18, [%[outptr6]], #16\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr7]], #32\n"
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr7]], #16\n"
- "ADD %[inptr], %[inptr], #384\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- : [av] "w" (av), [bv] "w" (bv)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21"
- );
- }
- } else {
- /* For ragged X, manually copy over the valid results. */
- if ((i+11) >= xmax) {
- for (int xi=0; xi<12; xi++) {
- if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
- outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
- outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
- outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
- outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
- outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
- outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta);
- outptr6++;
- *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta);
- outptr7++;
- }
- }
- inptr += 96;
- } else {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- // Rows 0-1
- "LDP q16, q17, [%[outptr0]]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDR q18, [%[outptr0], #32]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDP q19, q20, [%[outptr1]]\n"
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- "LDR q21, [%[outptr1], #32]\n"
- ASM_PREFETCH("[%[inptr], #768]")
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr]]\n"
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- "LDP q2, q3, [%[inptr], #32]\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "LDP q4, q5, [%[inptr], #64]\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #832]")
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr0]], #32\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q18, [%[outptr0]], #16\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #896]")
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr1]], #32\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr1]], #16\n"
-
- // Rows 2-3
- "LDP q16, q17, [%[outptr2]]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDR q18, [%[outptr2], #32]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDP q19, q20, [%[outptr3]]\n"
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- "LDR q21, [%[outptr3], #32]\n"
- ASM_PREFETCH("[%[inptr], #960]")
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr], #96]\n"
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- "LDP q2, q3, [%[inptr], #128]\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "LDP q4, q5, [%[inptr], #160]\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #1024]")
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr2]], #32\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q18, [%[outptr2]], #16\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #1088]")
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr3]], #32\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr3]], #16\n"
-
- // Rows 4-5
- ASM_PREFETCH("[%[outptr0], #80]")
- "LDP q16, q17, [%[outptr4]]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDR q18, [%[outptr4], #32]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDP q19, q20, [%[outptr5]]\n"
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- "LDR q21, [%[outptr5], #32]\n"
- ASM_PREFETCH("[%[outptr1], #80]")
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr], #192]\n"
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- "LDP q2, q3, [%[inptr], #224]\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "LDP q4, q5, [%[inptr], #256]\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- ASM_PREFETCH("[%[outptr2], #80]")
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr4]], #32\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q18, [%[outptr4]], #16\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- ASM_PREFETCH("[%[outptr3], #80]")
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr5]], #32\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr5]], #16\n"
-
- // Rows 6-7
- ASM_PREFETCH("[%[outptr4], #80]")
- "LDP q16, q17, [%[outptr6]]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDR q18, [%[outptr6], #32]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDP q19, q20, [%[outptr7]]\n"
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- "LDR q21, [%[outptr7], #32]\n"
- ASM_PREFETCH("[%[outptr5], #80]")
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr], #288]\n"
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- "LDP q2, q3, [%[inptr], #320]\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "LDP q4, q5, [%[inptr], #352]\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- ASM_PREFETCH("[%[outptr6], #128]")
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "STP q16, q17, [%[outptr6]], #32\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q18, [%[outptr6]], #16\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- ASM_PREFETCH("[%[outptr7], #128]")
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "STP q19, q20, [%[outptr7]], #32\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "STR q21, [%[outptr7]], #16\n"
- "ADD %[inptr], %[inptr], #384\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- : [av] "w" (av), [bv] "w" (bv)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21"
- );
- }
- }
- }
- }
-}
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp
deleted file mode 100644
index 0e638ee..0000000
--- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * Copyright (c) 2017-2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-// This should be possible on any AArch64 target, but some old compilers don't support __fp16 arguments.
-#if defined(__aarch64__) && defined(__ARM_FP16_ARGS)
-
-#include <arm_neon.h>
-
-template<>
-inline void MergeResults<12,8,false>(__fp16 *out, const float *in, int ldout, int y0, int ymax, int x0, int xmax, const __fp16 alpha, const __fp16 beta) {
- const float *inptr = in;
- prefetch_6x(inptr);
- prefetch_6x(inptr + 24);
-
- float32x4_t av = vdupq_n_f32(alpha);
- float32x4_t bv = vdupq_n_f32(beta);
-
- for (int y=y0; y<ymax; y+=8) {
- __fp16 *outptr0 = out + (y * ldout) + x0;
- __fp16 *outptr1 = outptr0 + ldout;
- __fp16 *outptr2 = outptr1 + ldout;
- __fp16 *outptr3 = outptr2 + ldout;
- __fp16 *outptr4 = outptr3 + ldout;
- __fp16 *outptr5 = outptr4 + ldout;
- __fp16 *outptr6 = outptr5 + ldout;
- __fp16 *outptr7 = outptr6 + ldout;
-
- prefetch_2x(outptr0);
- prefetch_2x(outptr1);
- prefetch_2x(outptr2);
- prefetch_2x(outptr3);
- prefetch_2x(outptr4);
- prefetch_2x(outptr5);
- prefetch_2x(outptr6);
- prefetch_2x(outptr7);
-
- for (int i=x0; i<xmax; i+=12) {
- __fp16 dummyres[12];
-
- /* Make sure we throw away results if Y isn't a multiple of 8.
- * We do this by pointing the result pointer at a dummy buffer
- * we later discard. */
- if ((y+7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- outptr1 = dummyres;
- // fall through
- case 5:
- outptr2 = dummyres;
- // fall through
- case 4:
- outptr3 = dummyres;
- // fall through
- case 3:
- outptr4 = dummyres;
- // fall through
- case 2:
- outptr5 = dummyres;
- // fall through
- case 1:
- outptr6 = dummyres;
- // fall through
- case 0:
- outptr7 = dummyres;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (beta == ((__fp16)0.0f)) {
- /* If beta==0, don't read the output. */
- /* For ragged X, manually copy over the valid results. */
- if ((i+11) >= xmax) {
- for (int xi=0; xi<12; xi++) {
- if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]);
- outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
- outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
- outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]);
- outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]);
- outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]);
- outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]);
- outptr6++;
- *outptr7 = (alpha * inptr[xi + 84]);
- outptr7++;
- }
- }
- inptr += 96;
- } else {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- // Rows 0-1
- "LDP q0, q1, [%[inptr]]\n"
- "LDP q2, q3, [%[inptr], #32]\n"
- "LDP q4, q5, [%[inptr], #64]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #768]")
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #832]")
- "FCVTN v16.4h, v16.4s\n"
- ASM_PREFETCH("[%[inptr], #896]")
- "FCVTN2 v16.8h, v17.4s\n"
- ASM_PREFETCH("[%[inptr], #960]")
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr0]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr0]], #8\n"
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr1]], #16\n"
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr1]], #8\n"
-
- // Rows 2-3
- "LDP q0, q1, [%[inptr], #96]\n"
- "LDP q2, q3, [%[inptr], #128]\n"
- "LDP q4, q5, [%[inptr], #160]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #1024]")
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- ASM_PREFETCH("[%[inptr], #1088]")
- "FCVTN v16.4h, v16.4s\n"
- ASM_PREFETCH("[%[outptr0], #64]")
- "FCVTN2 v16.8h, v17.4s\n"
- ASM_PREFETCH("[%[outptr1], #64]")
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr2]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr2]], #8\n"
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr3]], #16\n"
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr3]], #8\n"
-
- // Rows 4-5
- "LDP q0, q1, [%[inptr], #192]\n"
- "LDP q2, q3, [%[inptr], #224]\n"
- "LDP q4, q5, [%[inptr], #256]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- ASM_PREFETCH("[%[outptr2], #64]")
- "FCVTN v16.4h, v16.4s\n"
- ASM_PREFETCH("[%[outptr3], #64]")
- "FCVTN2 v16.8h, v17.4s\n"
- ASM_PREFETCH("[%[outptr4], #88]")
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr4]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr4]], #8\n"
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr5]], #16\n"
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr5]], #8\n"
-
- // Rows 6-7
- "LDP q0, q1, [%[inptr], #288]\n"
- "LDP q2, q3, [%[inptr], #320]\n"
- "LDP q4, q5, [%[inptr], #352]\n"
- "FMUL v16.4s, v0.4s, %[av].4s\n"
- "FMUL v17.4s, v1.4s, %[av].4s\n"
- ASM_PREFETCH("[%[outptr5], #64]")
- "FCVTN v16.4h, v16.4s\n"
- ASM_PREFETCH("[%[outptr6], #88]")
- "FCVTN2 v16.8h, v17.4s\n"
- ASM_PREFETCH("[%[outptr7], #88]")
- "FMUL v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr6]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr6]], #8\n"
- "FMUL v19.4s, v3.4s, %[av].4s\n"
- "FMUL v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr7]], #16\n"
- "FMUL v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr7]], #8\n"
- "ADD %[inptr], %[inptr], #384\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- : [av] "w" (av), [bv] "w" (bv)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21"
- );
- }
- } else {
- /* For ragged X, manually copy over the valid results. */
- if ((i+11) >= xmax) {
- for (int xi=0; xi<12; xi++) {
- if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
- outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
- outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
- outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
- outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
- outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
- outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta);
- outptr6++;
- *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta);
- outptr7++;
- }
- }
- inptr += 96;
- } else {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- // Rows 0-1
- "LDR q16, [%[outptr0]]\n"
- "FCVTL2 v17.4s, v16.8h\n"
- "LDR d18, [%[outptr0], #16]\n"
- "FCVTL v16.4s, v16.4h\n"
- "LDR q19, [%[outptr1]]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDR d21, [%[outptr1], #16]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr]]\n"
- "FCVTL v18.4s, v18.4h\n"
- "LDP q2, q3, [%[inptr], #32]\n"
- "FCVTL2 v20.4s, v19.8h\n"
- "LDP q4, q5, [%[inptr], #64]\n"
- "FCVTL v19.4s, v19.4h\n"
- ASM_PREFETCH("[%[inptr], #768]")
- "FCVTL v21.4s, v21.4h\n"
- ASM_PREFETCH("[%[inptr], #832]")
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- ASM_PREFETCH("[%[inptr], #896]")
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- ASM_PREFETCH("[%[inptr], #960]")
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "FCVTN v16.4h, v16.4s\n"
- "FCVTN2 v16.8h, v17.4s\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr0]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr0]], #8\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr1]], #16\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr1]], #8\n"
-
- // Rows 2-3
- "LDR q16, [%[outptr2]]\n"
- "FCVTL2 v17.4s, v16.8h\n"
- "LDR d18, [%[outptr2], #16]\n"
- "FCVTL v16.4s, v16.4h\n"
- "LDR q19, [%[outptr3]]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDR d21, [%[outptr3], #16]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr], #96]\n"
- "FCVTL v18.4s, v18.4h\n"
- "LDP q2, q3, [%[inptr], #128]\n"
- "FCVTL2 v20.4s, v19.8h\n"
- "LDP q4, q5, [%[inptr], #160]\n"
- "FCVTL v19.4s, v19.4h\n"
- ASM_PREFETCH("[%[inptr], #1024]")
- "FCVTL v21.4s, v21.4h\n"
- ASM_PREFETCH("[%[inptr], #1088]")
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- ASM_PREFETCH("[%[outptr0], #64]")
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- ASM_PREFETCH("[%[outptr1], #64]")
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "FCVTN v16.4h, v16.4s\n"
- "FCVTN2 v16.8h, v17.4s\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr2]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr2]], #8\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr3]], #16\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr3]], #8\n"
-
- // Rows 4-5
- "LDR q16, [%[outptr4]]\n"
- "FCVTL2 v17.4s, v16.8h\n"
- "LDR d18, [%[outptr4], #16]\n"
- "FCVTL v16.4s, v16.4h\n"
- "LDR q19, [%[outptr5]]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDR d21, [%[outptr5], #16]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr], #192]\n"
- "FCVTL v18.4s, v18.4h\n"
- "LDP q2, q3, [%[inptr], #224]\n"
- "FCVTL2 v20.4s, v19.8h\n"
- "LDP q4, q5, [%[inptr], #256]\n"
- "FCVTL v19.4s, v19.4h\n"
- ASM_PREFETCH("[%[outptr2], #64]")
- "FCVTL v21.4s, v21.4h\n"
- ASM_PREFETCH("[%[outptr3], #64]")
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- ASM_PREFETCH("[%[outptr4], #88]")
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "FCVTN v16.4h, v16.4s\n"
- "FCVTN2 v16.8h, v17.4s\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr4]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr4]], #8\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr5]], #16\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr5]], #8\n"
-
- // Rows 6-7
- "LDR q16, [%[outptr6]]\n"
- "FCVTL2 v17.4s, v16.8h\n"
- "LDR d18, [%[outptr6], #16]\n"
- "FCVTL v16.4s, v16.4h\n"
- "LDR q19, [%[outptr7]]\n"
- "FMUL v17.4s, v17.4s, %[bv].4s\n"
- "LDR d21, [%[outptr7], #16]\n"
- "FMUL v16.4s, v16.4s, %[bv].4s\n"
- "LDP q0, q1, [%[inptr], #288]\n"
- "FCVTL v18.4s, v18.4h\n"
- "LDP q2, q3, [%[inptr], #320]\n"
- "FCVTL2 v20.4s, v19.8h\n"
- "LDP q4, q5, [%[inptr], #352]\n"
- "FCVTL v19.4s, v19.4h\n"
- ASM_PREFETCH("[%[outptr5], #64]")
- "FCVTL v21.4s, v21.4h\n"
- ASM_PREFETCH("[%[outptr6], #88]")
- "FMUL v18.4s, v18.4s, %[bv].4s\n"
- ASM_PREFETCH("[%[outptr7], #88]")
- "FMUL v20.4s, v20.4s, %[bv].4s\n"
- "FMUL v19.4s, v19.4s, %[bv].4s\n"
- "FMUL v21.4s, v21.4s, %[bv].4s\n"
- "FMLA v16.4s, v0.4s, %[av].4s\n"
- "FMLA v17.4s, v1.4s, %[av].4s\n"
- "FCVTN v16.4h, v16.4s\n"
- "FCVTN2 v16.8h, v17.4s\n"
- "FMLA v18.4s, v2.4s, %[av].4s\n"
- "STR q16, [%[outptr6]], #16\n"
- "FCVTN v18.4h, v18.4s\n"
- "STR d18, [%[outptr6]], #8\n"
- "FMLA v19.4s, v3.4s, %[av].4s\n"
- "FMLA v20.4s, v4.4s, %[av].4s\n"
- "FCVTN v19.4h, v19.4s\n"
- "FCVTN2 v19.8h, v20.4s\n"
- "STR q19, [%[outptr7]], #16\n"
- "FMLA v21.4s, v5.4s, %[av].4s\n"
- "FCVTN v21.4h, v21.4s\n"
- "STR d21, [%[outptr7]], #8\n"
- "ADD %[inptr], %[inptr], #384\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- : [av] "w" (av), [bv] "w" (bv)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21"
- );
- }
- }
- }
- }
-}
-
-#endif // __aarch64__ && __ARM_FP16_ARGS
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp
new file mode 100644
index 0000000..7bfab41
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp
@@ -0,0 +1,2074 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
+
+template<>
+void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const __fp16 *bias, Activation act, bool append)
+{
+ const __fp16 *inptr = in;
+ __fp16 nullbias[24] = { 0 };
+ __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity());
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<__fp16>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (24 * sizeof(__fp16)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
+ __fp16 *outptr0 = out + (y * ldout) + x0;
+ __fp16 *outptr1 = outptr0 + ldout;
+ __fp16 *outptr2 = outptr1 + ldout;
+ __fp16 *outptr3 = outptr2 + ldout;
+ __fp16 *outptr4 = outptr3 + ldout;
+ __fp16 *outptr5 = outptr4 + ldout;
+ __fp16 *outptr6 = outptr5 + ldout;
+ __fp16 *outptr7 = outptr6 + ldout;
+
+ const int height = ymax - y;
+
+ for (int i=x0; i<xmax; i+=24)
+ {
+ if (append)
+ {
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval);
+ outptr1++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "str q13, [%[outptr1]]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval);
+ outptr2++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr1]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr2]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval);
+ outptr3++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q11, [%[outptr3]]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval);
+ outptr4++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q11, [%[outptr3]]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr4]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval);
+ outptr4++;
+ *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval);
+ outptr5++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q9, [%[outptr5]]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr3]]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr4]]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "str q17, [%[outptr5]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval);
+ outptr4++;
+ *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval);
+ outptr5++;
+ *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + *outptr6)), maxval);
+ outptr6++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q9, [%[outptr5]]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr3]]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q4, [%[outptr6]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr4]]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "str q17, [%[outptr5]]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr6]]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval);
+ outptr4++;
+ *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval);
+ outptr5++;
+ *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + *outptr6)), maxval);
+ outptr6++;
+ *outptr7 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 168] + *outptr7)), maxval);
+ outptr7++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q9, [%[outptr5]]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr3]]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q4, [%[outptr6]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "fadd v10.8h, v10.8h, v2.8h\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr4]]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "fadd v11.8h, v11.8h, v3.8h\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "fadd v12.8h, v12.8h, v4.8h\n"
+ "ldr q7, [%[outptr7]]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "ldr q15, [%[inptr], #0x150]\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "ldr q8, [%[outptr7], #0x10]\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "str q17, [%[outptr5]]\n"
+ "fadd v13.8h, v13.8h, v5.8h\n"
+ "ldr q16, [%[inptr], #0x160]\n"
+ "fadd v14.8h, v14.8h, v6.8h\n"
+ "ldr q9, [%[outptr7], #0x20]\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x170]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "fadd v15.8h, v15.8h, v7.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q12, [%[outptr6]]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "fadd v16.8h, v16.8h, v8.8h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v17.8h, v17.8h, v9.8h\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "str q15, [%[outptr7]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q16, [%[outptr7], #0x10]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "str q17, [%[outptr7], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ else
+ {
+ const __fp16 *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval);
+ outptr1++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fadd v16.8h, v16.8h, v2.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "fadd v17.8h, v17.8h, v3.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fadd v18.8h, v18.8h, v4.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr1]]\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval);
+ outptr2++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.8h, v16.8h, v2.8h\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.8h, v17.8h, v3.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fadd v18.8h, v18.8h, v4.8h\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.8h, v19.8h, v2.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "fadd v20.8h, v20.8h, v3.8h\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fadd v13.8h, v13.8h, v4.8h\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "str q19, [%[outptr2]]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval);
+ outptr3++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.8h, v16.8h, v2.8h\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.8h, v17.8h, v3.8h\n"
+ "fadd v18.8h, v18.8h, v4.8h\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.8h, v19.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fadd v20.8h, v20.8h, v3.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr1]]\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fadd v13.8h, v13.8h, v4.8h\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fadd v14.8h, v14.8h, v2.8h\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fadd v15.8h, v15.8h, v3.8h\n"
+ "fadd v16.8h, v16.8h, v4.8h\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr3]]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval);
+ outptr4++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.8h, v16.8h, v2.8h\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.8h, v17.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.8h, v18.8h, v4.8h\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.8h, v19.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.8h, v20.8h, v3.8h\n"
+ "fadd v13.8h, v13.8h, v4.8h\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.8h, v14.8h, v2.8h\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fadd v15.8h, v15.8h, v3.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "str q19, [%[outptr2]]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fadd v16.8h, v16.8h, v4.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fadd v17.8h, v17.8h, v2.8h\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q14, [%[outptr3]]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fadd v18.8h, v18.8h, v3.8h\n"
+ "fadd v19.8h, v19.8h, v4.8h\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "str q17, [%[outptr4]]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval);
+ outptr4++;
+ *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval);
+ outptr5++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.8h, v16.8h, v2.8h\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.8h, v17.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.8h, v18.8h, v4.8h\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.8h, v19.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.8h, v20.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fadd v13.8h, v13.8h, v4.8h\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.8h, v14.8h, v2.8h\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fadd v15.8h, v15.8h, v3.8h\n"
+ "fadd v16.8h, v16.8h, v4.8h\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fadd v17.8h, v17.8h, v2.8h\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "fadd v18.8h, v18.8h, v3.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr3]]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fadd v19.8h, v19.8h, v4.8h\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fadd v20.8h, v20.8h, v2.8h\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "str q17, [%[outptr4]]\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "fadd v13.8h, v13.8h, v3.8h\n"
+ "fadd v14.8h, v14.8h, v4.8h\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q20, [%[outptr5]]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval);
+ outptr4++;
+ *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval);
+ outptr5++;
+ *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + biasptr[xi])), maxval);
+ outptr6++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.8h, v16.8h, v2.8h\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.8h, v17.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.8h, v18.8h, v4.8h\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.8h, v19.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.8h, v20.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fadd v13.8h, v13.8h, v4.8h\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.8h, v14.8h, v2.8h\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fadd v15.8h, v15.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fadd v16.8h, v16.8h, v4.8h\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fadd v17.8h, v17.8h, v2.8h\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q14, [%[outptr3]]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "fadd v18.8h, v18.8h, v3.8h\n"
+ "fadd v19.8h, v19.8h, v4.8h\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "fadd v20.8h, v20.8h, v2.8h\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "fadd v13.8h, v13.8h, v3.8h\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "str q17, [%[outptr4]]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "fadd v14.8h, v14.8h, v4.8h\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "fadd v15.8h, v15.8h, v2.8h\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q20, [%[outptr5]]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fadd v16.8h, v16.8h, v3.8h\n"
+ "fadd v17.8h, v17.8h, v4.8h\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "str q15, [%[outptr6]]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+23) >= xmax)
+ {
+ for (int xi=0; xi<23; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval);
+ outptr0++;
+ *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval);
+ outptr1++;
+ *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval);
+ outptr2++;
+ *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval);
+ outptr3++;
+ *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval);
+ outptr4++;
+ *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval);
+ outptr5++;
+ *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + biasptr[xi])), maxval);
+ outptr6++;
+ *outptr7 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 168] + biasptr[xi])), maxval);
+ outptr7++;
+ }
+ }
+ inptr += 192;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "dup v0.8h, %[maxval].h[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.8h, %[minval].h[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd v13.8h, v13.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.8h, v14.8h, v3.8h\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.8h, v15.8h, v4.8h\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.8h, v16.8h, v2.8h\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.8h, v17.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.8h, v18.8h, v4.8h\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.8h, v19.8h, v2.8h\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.8h, v20.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fadd v13.8h, v13.8h, v4.8h\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.8h, v14.8h, v2.8h\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fadd v15.8h, v15.8h, v3.8h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fadd v16.8h, v16.8h, v4.8h\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fadd v17.8h, v17.8h, v2.8h\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "str q14, [%[outptr3]]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "fadd v18.8h, v18.8h, v3.8h\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fadd v19.8h, v19.8h, v4.8h\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "fadd v20.8h, v20.8h, v2.8h\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "fadd v13.8h, v13.8h, v3.8h\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "str q17, [%[outptr4]]\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "fadd v14.8h, v14.8h, v4.8h\n"
+ "ldr q18, [%[inptr], #0x150]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "fadd v15.8h, v15.8h, v2.8h\n"
+ "ldr q19, [%[inptr], #0x160]\n"
+ "fadd v16.8h, v16.8h, v3.8h\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "str q20, [%[outptr5]]\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "ldr q20, [%[inptr], #0x170]\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fadd v17.8h, v17.8h, v4.8h\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fadd v18.8h, v18.8h, v2.8h\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fadd v19.8h, v19.8h, v3.8h\n"
+ "str q15, [%[outptr6]]\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fadd v20.8h, v20.8h, v4.8h\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "str q18, [%[outptr7]]\n"
+ "str q19, [%[outptr7], #0x10]\n"
+ "str q20, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ }
+ }
+}
+
+#endif // __aarch64__ && (FP16_KERNELS || __ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp
index fcdca59..0c5aa7e 100644
--- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp
@@ -26,11 +26,33 @@
#ifdef __aarch64__
template<>
-inline void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta)
+void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append)
{
const float *inptr = in;
+ float nullbias[12] = { 0 };
+ float minval = - std::numeric_limits<float>::infinity();
+ float maxval = std::numeric_limits<float>::infinity();
- for (int y=y0; y<ymax; y+=8) {
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (12 * sizeof(float)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
float *outptr0 = out + (y * ldout) + x0;
float *outptr1 = outptr0 + ldout;
float *outptr2 = outptr1 + ldout;
@@ -42,19 +64,21 @@
const int height = ymax - y;
- for (int i=x0; i<xmax; i+=12) {
- if (beta==0.0f)
+ for (int i=x0; i<xmax; i+=12)
+ {
+ if (append)
{
- switch(height) {
+ switch(height)
+ {
case 1:
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
}
}
@@ -62,23 +86,34 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x10]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr0], #0x10]\n"
- "ldr q6, [%[inptr], #0x20]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"add %[inptr], %[inptr], #0x180\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "str q12, [%[outptr0], #0x20]\n"
"add %[outptr0], %[outptr0], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -88,13 +123,13 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval);
outptr1++;
}
}
@@ -102,35 +137,55 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x30]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q6, [%[inptr], #0x10]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x10]\n"
- "ldr q7, [%[inptr], #0x40]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x10]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q5, [%[inptr], #0x50]\n"
- "add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "add %[outptr1], %[outptr1], #0x30\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -140,15 +195,15 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval);
outptr2++;
}
}
@@ -156,47 +211,76 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x30]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q7, [%[inptr], #0x10]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"str q11, [%[outptr0], #0x10]\n"
- "ldr q4, [%[inptr], #0x40]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr1], #0x10]\n"
- "ldr q5, [%[inptr], #0x70]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr2], #0x10]\n"
- "ldr q6, [%[inptr], #0x20]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
- "ldr q7, [%[inptr], #0x50]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x20]\n"
- "ldr q4, [%[inptr], #0x80]\n"
- "add %[outptr1], %[outptr1], #0x30\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x20]\n"
- "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -206,17 +290,17 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval);
outptr3++;
}
}
@@ -224,58 +308,96 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x30]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q4, [%[inptr], #0x10]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x10]\n"
- "ldr q5, [%[inptr], #0x40]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x10]\n"
- "ldr q6, [%[inptr], #0x70]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2], #0x10]\n"
- "ldr q7, [%[inptr], #0xa0]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x10]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q5, [%[inptr], #0x50]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "ldr q6, [%[inptr], #0x80]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
"add %[outptr1], %[outptr1], #0x30\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
"str q10, [%[outptr2], #0x20]\n"
- "ldr q7, [%[inptr], #0xb0]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x20]\n"
- "add %[outptr3], %[outptr3], #0x30\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q11, [%[outptr3]]\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -285,19 +407,19 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval);
outptr4++;
}
}
@@ -305,70 +427,117 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q6, [%[inptr], #0x60]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q5, [%[inptr], #0x10]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr0], #0x10]\n"
- "ldr q6, [%[inptr], #0x40]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr1], #0x10]\n"
- "ldr q7, [%[inptr], #0x70]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr2], #0x10]\n"
- "ldr q4, [%[inptr], #0xa0]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr3], #0x10]\n"
- "ldr q5, [%[inptr], #0xd0]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr4], #0x10]\n"
- "ldr q6, [%[inptr], #0x20]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
- "ldr q7, [%[inptr], #0x50]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x20]\n"
- "ldr q4, [%[inptr], #0x80]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
"add %[outptr1], %[outptr1], #0x30\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x20]\n"
- "ldr q5, [%[inptr], #0xb0]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr3], #0x20]\n"
- "ldr q6, [%[inptr], #0xe0]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
"add %[outptr3], %[outptr3], #0x30\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr4], #0x20]\n"
- "add %[outptr4], %[outptr4], #0x30\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q15, [%[outptr4], #0x10]\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -378,21 +547,21 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval);
outptr5++;
}
}
@@ -400,82 +569,138 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q6, [%[inptr], #0x60]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q5, [%[inptr], #0xf0]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5]]\n"
- "ldr q6, [%[inptr], #0x10]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x10]\n"
- "ldr q7, [%[inptr], #0x40]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x10]\n"
- "ldr q4, [%[inptr], #0x70]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x10]\n"
- "ldr q5, [%[inptr], #0xa0]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr3], #0x10]\n"
- "ldr q6, [%[inptr], #0xd0]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr4], #0x10]\n"
- "ldr q7, [%[inptr], #0x100]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr5], #0x10]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q5, [%[inptr], #0x50]\n"
- "add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "ldr q6, [%[inptr], #0x80]\n"
- "add %[outptr1], %[outptr1], #0x30\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q9, [%[outptr5]]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
"str q10, [%[outptr2], #0x20]\n"
- "ldr q7, [%[inptr], #0xb0]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x20]\n"
- "ldr q4, [%[inptr], #0xe0]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "str q17, [%[outptr5]]\n"
"add %[outptr3], %[outptr3], #0x30\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4], #0x20]\n"
- "ldr q5, [%[inptr], #0x110]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q10, [%[outptr5], #0x10]\n"
"add %[outptr4], %[outptr4], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "str q11, [%[outptr5], #0x20]\n"
"add %[outptr5], %[outptr5], #0x30\n"
"add %[inptr], %[inptr], #0x180\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -485,23 +710,23 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval);
outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]);
+ *outptr6 = std::min(std::max(minval, inptr[xi + 72] + *outptr6), maxval);
outptr6++;
}
}
@@ -509,94 +734,159 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q6, [%[inptr], #0x60]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q5, [%[inptr], #0xf0]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5]]\n"
- "ldr q6, [%[inptr], #0x120]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6]]\n"
- "ldr q7, [%[inptr], #0x10]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"str q11, [%[outptr0], #0x10]\n"
- "ldr q4, [%[inptr], #0x40]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr1], #0x10]\n"
- "ldr q5, [%[inptr], #0x70]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr2], #0x10]\n"
- "ldr q6, [%[inptr], #0xa0]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr3], #0x10]\n"
- "ldr q7, [%[inptr], #0xd0]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr4], #0x10]\n"
- "ldr q4, [%[inptr], #0x100]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr5], #0x10]\n"
- "ldr q5, [%[inptr], #0x130]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr6], #0x10]\n"
- "ldr q6, [%[inptr], #0x20]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
- "ldr q7, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q9, [%[outptr5]]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q4, [%[outptr6]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x20]\n"
- "ldr q4, [%[inptr], #0x80]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
"add %[outptr1], %[outptr1], #0x30\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x20]\n"
- "ldr q5, [%[inptr], #0xb0]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr3], #0x20]\n"
- "ldr q6, [%[inptr], #0xe0]\n"
- "add %[outptr3], %[outptr3], #0x30\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr4], #0x20]\n"
- "ldr q7, [%[inptr], #0x110]\n"
- "add %[outptr4], %[outptr4], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
"str q11, [%[outptr5], #0x20]\n"
- "ldr q4, [%[inptr], #0x140]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr6]]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
"add %[outptr5], %[outptr5], #0x30\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr6], #0x20]\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
"add %[outptr6], %[outptr6], #0x30\n"
"add %[inptr], %[inptr], #0x180\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -607,25 +897,25 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval);
outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]);
+ *outptr6 = std::min(std::max(minval, inptr[xi + 72] + *outptr6), maxval);
outptr6++;
- *outptr7 = (alpha * inptr[xi + 84]);
+ *outptr7 = std::min(std::max(minval, inptr[xi + 84] + *outptr7), maxval);
outptr7++;
}
}
@@ -633,105 +923,179 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q4, [%[inptr]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[outptr0]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q10, [%[inptr]]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q6, [%[inptr], #0x60]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q5, [%[inptr], #0xf0]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5]]\n"
- "ldr q6, [%[inptr], #0x120]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6]]\n"
- "ldr q7, [%[inptr], #0x150]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr7]]\n"
- "ldr q4, [%[inptr], #0x10]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x10]\n"
- "ldr q5, [%[inptr], #0x40]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x10]\n"
- "ldr q6, [%[inptr], #0x70]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2], #0x10]\n"
- "ldr q7, [%[inptr], #0xa0]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x10]\n"
- "ldr q4, [%[inptr], #0xd0]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4], #0x10]\n"
- "ldr q5, [%[inptr], #0x100]\n"
- "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5], #0x10]\n"
- "ldr q6, [%[inptr], #0x130]\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6], #0x10]\n"
- "ldr q7, [%[inptr], #0x160]\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr7], #0x10]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q5, [%[inptr], #0x50]\n"
- "add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "ldr q6, [%[inptr], #0x80]\n"
- "add %[outptr1], %[outptr1], #0x30\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr1]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "ldr q8, [%[outptr2]]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr3]]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q9, [%[outptr5]]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
"str q10, [%[outptr2], #0x20]\n"
- "ldr q7, [%[inptr], #0xb0]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q4, [%[outptr6]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "fadd v10.4s, v10.4s, v2.4s\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "fadd v11.4s, v11.4s, v3.4s\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "fadd v12.4s, v12.4s, v4.4s\n"
+ "ldr q7, [%[outptr7]]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "ldr q15, [%[inptr], #0x150]\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "ldr q8, [%[outptr7], #0x10]\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "fadd v13.4s, v13.4s, v5.4s\n"
+ "ldr q16, [%[inptr], #0x160]\n"
+ "fadd v14.4s, v14.4s, v6.4s\n"
+ "ldr q9, [%[outptr7], #0x20]\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x170]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "fadd v15.4s, v15.4s, v7.4s\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q12, [%[outptr6]]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "fadd v16.4s, v16.4s, v8.4s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd v17.4s, v17.4s, v9.4s\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x20]\n"
- "ldr q4, [%[inptr], #0xe0]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "str q15, [%[outptr7]]\n"
"add %[outptr3], %[outptr3], #0x30\n"
- "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4], #0x20]\n"
- "ldr q5, [%[inptr], #0x110]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "str q16, [%[outptr7], #0x10]\n"
"add %[outptr4], %[outptr4], #0x30\n"
- "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5], #0x20]\n"
- "ldr q6, [%[inptr], #0x140]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "str q17, [%[outptr7], #0x20]\n"
"add %[outptr5], %[outptr5], #0x30\n"
- "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6], #0x20]\n"
- "ldr q7, [%[inptr], #0x170]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
"add %[outptr6], %[outptr6], #0x30\n"
- "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr7], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
"add %[outptr7], %[outptr7], #0x30\n"
"add %[inptr], %[inptr], #0x180\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -742,16 +1106,23 @@
}
else
{
- switch(height) {
+ const float *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
case 1:
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
}
}
@@ -759,29 +1130,34 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr0], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x10]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr0], #0x10]\n"
- "ldr q10, [%[outptr0], #0x20]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x20]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
- "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "ldr q15, [%[inptr], #0x20]\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -791,13 +1167,13 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval);
outptr1++;
}
}
@@ -805,47 +1181,52 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr1]]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q10, [%[outptr0], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x10]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x10]\n"
- "ldr q11, [%[outptr1], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x40]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x10]\n"
- "ldr q8, [%[outptr0], #0x20]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q9, [%[outptr1], #0x20]\n"
- "add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x50]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "add %[outptr1], %[outptr1], #0x30\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fadd v16.4s, v16.4s, v2.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "fadd v17.4s, v17.4s, v3.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fadd v18.4s, v18.4s, v4.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -855,15 +1236,15 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval);
outptr2++;
}
}
@@ -871,65 +1252,70 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr1]]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q10, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q11, [%[outptr0], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x10]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr0], #0x10]\n"
- "ldr q8, [%[outptr1], #0x10]\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.4s, v16.4s, v2.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x40]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr1], #0x10]\n"
- "ldr q9, [%[outptr2], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x70]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr2], #0x10]\n"
- "ldr q10, [%[outptr0], #0x20]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x20]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
- "ldr q11, [%[outptr1], #0x20]\n"
- "add %[outptr0], %[outptr0], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x50]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x20]\n"
- "ldr q8, [%[outptr2], #0x20]\n"
- "add %[outptr1], %[outptr1], #0x30\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x80]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x20]\n"
- "add %[outptr2], %[outptr2], #0x30\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.4s, v17.4s, v3.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fadd v18.4s, v18.4s, v4.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.4s, v19.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fadd v20.4s, v20.4s, v3.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fadd v13.4s, v13.4s, v4.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "str q19, [%[outptr2]]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -939,17 +1325,17 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval);
outptr3++;
}
}
@@ -957,82 +1343,87 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr1]]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q10, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q11, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q8, [%[outptr0], #0x10]\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.4s, v16.4s, v2.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x10]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x10]\n"
- "ldr q9, [%[outptr1], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x40]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x10]\n"
- "ldr q10, [%[outptr2], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x70]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2], #0x10]\n"
- "ldr q11, [%[outptr3], #0x10]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0xa0]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x10]\n"
- "ldr q8, [%[outptr0], #0x20]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q9, [%[outptr1], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.4s, v17.4s, v3.4s\n"
+ "fadd v18.4s, v18.4s, v4.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.4s, v19.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fadd v20.4s, v20.4s, v3.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x50]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "ldr q10, [%[outptr2], #0x20]\n"
- "add %[outptr1], %[outptr1], #0x30\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x80]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2], #0x20]\n"
- "ldr q11, [%[outptr3], #0x20]\n"
- "add %[outptr2], %[outptr2], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0xb0]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x20]\n"
- "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fadd v13.4s, v13.4s, v4.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fadd v14.4s, v14.4s, v2.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fadd v15.4s, v15.4s, v3.4s\n"
+ "fadd v16.4s, v16.4s, v4.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr3]]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -1042,19 +1433,19 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval);
outptr4++;
}
}
@@ -1062,100 +1453,105 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q10, [%[outptr2]]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q11, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q8, [%[outptr4]]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q9, [%[outptr0], #0x10]\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.4s, v16.4s, v2.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x10]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr0], #0x10]\n"
- "ldr q10, [%[outptr1], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x40]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr1], #0x10]\n"
- "ldr q11, [%[outptr2], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x70]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr2], #0x10]\n"
- "ldr q8, [%[outptr3], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xa0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr3], #0x10]\n"
- "ldr q9, [%[outptr4], #0x10]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0xd0]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr4], #0x10]\n"
- "ldr q10, [%[outptr0], #0x20]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x20]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
- "ldr q11, [%[outptr1], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.4s, v17.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.4s, v18.4s, v4.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.4s, v19.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x50]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x20]\n"
- "ldr q8, [%[outptr2], #0x20]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.4s, v20.4s, v3.4s\n"
+ "fadd v13.4s, v13.4s, v4.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.4s, v14.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fadd v15.4s, v15.4s, v3.4s\n"
"add %[outptr1], %[outptr1], #0x30\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x80]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x20]\n"
- "ldr q9, [%[outptr3], #0x20]\n"
- "add %[outptr2], %[outptr2], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0xb0]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr3], #0x20]\n"
- "ldr q10, [%[outptr4], #0x20]\n"
- "add %[outptr3], %[outptr3], #0x30\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0xe0]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr4], #0x20]\n"
- "add %[outptr4], %[outptr4], #0x30\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "str q19, [%[outptr2]]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fadd v16.4s, v16.4s, v4.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fadd v17.4s, v17.4s, v2.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q14, [%[outptr3]]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fadd v18.4s, v18.4s, v3.4s\n"
+ "fadd v19.4s, v19.4s, v4.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "str q17, [%[outptr4]]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -1165,21 +1561,21 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval);
outptr5++;
}
}
@@ -1187,118 +1583,123 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q10, [%[outptr2]]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q11, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q8, [%[outptr4]]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q9, [%[outptr5]]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0xf0]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5]]\n"
- "ldr q10, [%[outptr0], #0x10]\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.4s, v16.4s, v2.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x10]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x10]\n"
- "ldr q11, [%[outptr1], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x40]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x10]\n"
- "ldr q8, [%[outptr2], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x70]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x10]\n"
- "ldr q9, [%[outptr3], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0xa0]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr3], #0x10]\n"
- "ldr q10, [%[outptr4], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0xd0]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr4], #0x10]\n"
- "ldr q11, [%[outptr5], #0x10]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x100]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr5], #0x10]\n"
- "ldr q8, [%[outptr0], #0x20]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q9, [%[outptr1], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.4s, v17.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.4s, v18.4s, v4.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.4s, v19.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x50]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "ldr q10, [%[outptr2], #0x20]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.4s, v20.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fadd v13.4s, v13.4s, v4.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.4s, v14.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
"add %[outptr1], %[outptr1], #0x30\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x80]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2], #0x20]\n"
- "ldr q11, [%[outptr3], #0x20]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fadd v15.4s, v15.4s, v3.4s\n"
+ "fadd v16.4s, v16.4s, v4.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fadd v17.4s, v17.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "fadd v18.4s, v18.4s, v3.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0xb0]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x20]\n"
- "ldr q8, [%[outptr4], #0x20]\n"
- "add %[outptr3], %[outptr3], #0x30\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xe0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4], #0x20]\n"
- "ldr q9, [%[outptr5], #0x20]\n"
- "add %[outptr4], %[outptr4], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x110]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5], #0x20]\n"
- "add %[outptr5], %[outptr5], #0x30\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr3]]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fadd v19.4s, v19.4s, v4.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fadd v20.4s, v20.4s, v2.4s\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "str q17, [%[outptr4]]\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fadd v13.4s, v13.4s, v3.4s\n"
+ "fadd v14.4s, v14.4s, v4.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q20, [%[outptr5]]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -1308,23 +1709,23 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval);
outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta);
+ *outptr6 = std::min(std::max(minval, inptr[xi + 72] + biasptr[xi]), maxval);
outptr6++;
}
}
@@ -1332,136 +1733,141 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q10, [%[outptr2]]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q11, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q8, [%[outptr4]]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q9, [%[outptr5]]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0xf0]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5]]\n"
- "ldr q10, [%[outptr6]]\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.4s, v16.4s, v2.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x120]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6]]\n"
- "ldr q11, [%[outptr0], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x10]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr0], #0x10]\n"
- "ldr q8, [%[outptr1], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x40]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr1], #0x10]\n"
- "ldr q9, [%[outptr2], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x70]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr2], #0x10]\n"
- "ldr q10, [%[outptr3], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0xa0]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr3], #0x10]\n"
- "ldr q11, [%[outptr4], #0x10]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0xd0]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr4], #0x10]\n"
- "ldr q8, [%[outptr5], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x100]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr5], #0x10]\n"
- "ldr q9, [%[outptr6], #0x10]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x130]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr6], #0x10]\n"
- "ldr q10, [%[outptr0], #0x20]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x20]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr0], #0x20]\n"
- "ldr q11, [%[outptr1], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.4s, v17.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.4s, v18.4s, v4.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.4s, v19.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x50]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr1], #0x20]\n"
- "ldr q8, [%[outptr2], #0x20]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.4s, v20.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fadd v13.4s, v13.4s, v4.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.4s, v14.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
"add %[outptr1], %[outptr1], #0x30\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x80]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr2], #0x20]\n"
- "ldr q9, [%[outptr3], #0x20]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fadd v15.4s, v15.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fadd v16.4s, v16.4s, v4.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fadd v17.4s, v17.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0xb0]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr3], #0x20]\n"
- "ldr q10, [%[outptr4], #0x20]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q14, [%[outptr3]]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "fadd v18.4s, v18.4s, v3.4s\n"
+ "fadd v19.4s, v19.4s, v4.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "fadd v20.4s, v20.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "fadd v13.4s, v13.4s, v3.4s\n"
"add %[outptr3], %[outptr3], #0x30\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0xe0]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr4], #0x20]\n"
- "ldr q11, [%[outptr5], #0x20]\n"
- "add %[outptr4], %[outptr4], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x110]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr5], #0x20]\n"
- "ldr q8, [%[outptr6], #0x20]\n"
- "add %[outptr5], %[outptr5], #0x30\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x140]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr6], #0x20]\n"
- "add %[outptr6], %[outptr6], #0x30\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "str q17, [%[outptr4]]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "fadd v14.4s, v14.4s, v4.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fadd v15.4s, v15.4s, v2.4s\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q20, [%[outptr5]]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fadd v16.4s, v16.4s, v3.4s\n"
+ "fadd v17.4s, v17.4s, v4.4s\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "str q15, [%[outptr6]]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
@@ -1472,25 +1878,25 @@
{
if ((i+11) >= xmax)
{
- for (int xi=0; xi<12; xi++)
+ for (int xi=0; xi<11; xi++)
{
if ((i+xi) < xmax)
{
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval);
outptr0++;
- *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval);
outptr1++;
- *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval);
outptr2++;
- *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval);
outptr3++;
- *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval);
outptr4++;
- *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
+ *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval);
outptr5++;
- *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta);
+ *outptr6 = std::min(std::max(minval, inptr[xi + 72] + biasptr[xi]), maxval);
outptr6++;
- *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta);
+ *outptr7 = std::min(std::max(minval, inptr[xi + 84] + biasptr[xi]), maxval);
outptr7++;
}
}
@@ -1498,153 +1904,158 @@
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
- "ldr q8, [%[outptr0]]\n"
+ "dup v0.4s, %[maxval].s[0]\n"
+ "ldr q2, [%[biasptr]]\n"
+ "dup v1.4s, %[minval].s[0]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr]]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0]]\n"
- "ldr q9, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x30]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1]]\n"
- "ldr q10, [%[outptr2]]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x60]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2]]\n"
- "ldr q11, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x90]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3]]\n"
- "ldr q8, [%[outptr4]]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xc0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4]]\n"
- "ldr q9, [%[outptr5]]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0xf0]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5]]\n"
- "ldr q10, [%[outptr6]]\n"
+ "fadd v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd v14.4s, v14.4s, v3.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "fadd v15.4s, v15.4s, v4.4s\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "fadd v16.4s, v16.4s, v2.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x120]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6]]\n"
- "ldr q11, [%[outptr7]]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x150]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr7]]\n"
- "ldr q8, [%[outptr0], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x10]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x10]\n"
- "ldr q9, [%[outptr1], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x40]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x10]\n"
- "ldr q10, [%[outptr2], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x70]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2], #0x10]\n"
- "ldr q11, [%[outptr3], #0x10]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0xa0]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x10]\n"
- "ldr q8, [%[outptr4], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xd0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4], #0x10]\n"
- "ldr q9, [%[outptr5], #0x10]\n"
- "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x100]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5], #0x10]\n"
- "ldr q10, [%[outptr6], #0x10]\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x130]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6], #0x10]\n"
- "ldr q11, [%[outptr7], #0x10]\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x160]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr7], #0x10]\n"
- "ldr q8, [%[outptr0], #0x20]\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0x20]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr0], #0x20]\n"
- "ldr q9, [%[outptr1], #0x20]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "fadd v17.4s, v17.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fadd v18.4s, v18.4s, v4.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "fadd v19.4s, v19.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
"add %[outptr0], %[outptr0], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x50]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr1], #0x20]\n"
- "ldr q10, [%[outptr2], #0x20]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "fadd v20.4s, v20.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fadd v13.4s, v13.4s, v4.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "fadd v14.4s, v14.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
"add %[outptr1], %[outptr1], #0x30\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x80]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr2], #0x20]\n"
- "ldr q11, [%[outptr3], #0x20]\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q19, [%[outptr2]]\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "fadd v15.4s, v15.4s, v3.4s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fadd v16.4s, v16.4s, v4.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "fadd v17.4s, v17.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
"add %[outptr2], %[outptr2], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0xb0]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr3], #0x20]\n"
- "ldr q8, [%[outptr4], #0x20]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "str q14, [%[outptr3]]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "fadd v18.4s, v18.4s, v3.4s\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fadd v19.4s, v19.4s, v4.4s\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "fadd v20.4s, v20.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "fadd v13.4s, v13.4s, v3.4s\n"
"add %[outptr3], %[outptr3], #0x30\n"
- "fmul v8.4s, v8.4s, %[beta].s[0]\n"
- "ldr q4, [%[inptr], #0xe0]\n"
- "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
- "str q8, [%[outptr4], #0x20]\n"
- "ldr q9, [%[outptr5], #0x20]\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "str q17, [%[outptr4]]\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "fadd v14.4s, v14.4s, v4.4s\n"
+ "ldr q18, [%[inptr], #0x150]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "fadd v15.4s, v15.4s, v2.4s\n"
+ "ldr q19, [%[inptr], #0x160]\n"
+ "fadd v16.4s, v16.4s, v3.4s\n"
"add %[outptr4], %[outptr4], #0x30\n"
- "fmul v9.4s, v9.4s, %[beta].s[0]\n"
- "ldr q5, [%[inptr], #0x110]\n"
- "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
- "str q9, [%[outptr5], #0x20]\n"
- "ldr q10, [%[outptr6], #0x20]\n"
- "add %[outptr5], %[outptr5], #0x30\n"
- "fmul v10.4s, v10.4s, %[beta].s[0]\n"
- "ldr q6, [%[inptr], #0x140]\n"
- "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
- "str q10, [%[outptr6], #0x20]\n"
- "ldr q11, [%[outptr7], #0x20]\n"
- "add %[outptr6], %[outptr6], #0x30\n"
- "fmul v11.4s, v11.4s, %[beta].s[0]\n"
- "ldr q7, [%[inptr], #0x170]\n"
- "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
- "str q11, [%[outptr7], #0x20]\n"
- "add %[outptr7], %[outptr7], #0x30\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "str q20, [%[outptr5]]\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "ldr q20, [%[inptr], #0x170]\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
"add %[inptr], %[inptr], #0x180\n"
+ "fadd v17.4s, v17.4s, v4.4s\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fadd v18.4s, v18.4s, v2.4s\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fadd v19.4s, v19.4s, v3.4s\n"
+ "str q15, [%[outptr6]]\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fadd v20.4s, v20.4s, v4.4s\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "str q18, [%[outptr7]]\n"
+ "str q19, [%[outptr7], #0x10]\n"
+ "str q20, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr)
- : [alpha] "w" (alpha), [beta] "w" (beta)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
);
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp
deleted file mode 100644
index 60cc2f3..0000000
--- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp
+++ /dev/null
@@ -1,363 +0,0 @@
-/*
- * Copyright (c) 2017-2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-// AArch64 only, and either the FP16_KERNELS option set or the target explicitly supports FP16 vectors.
-#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
-
-template<>
-inline void MergeResults<24, 8>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax,
- const int x0, const int xmax, const __fp16 alpha, const __fp16 beta) {
- const __fp16 *inptr = in;
- prefetch_6x(inptr);
- prefetch_6x(inptr + 48);
-
- float16x8_t va = vdupq_n_f16(alpha);
- float16x8_t vb = vdupq_n_f16(beta);
-
- for (int y=y0; y<ymax; y+=8) {
- __fp16 *outptr0 = out + (y * ldout) + x0;
- __fp16 *outptr1 = outptr0 + ldout;
- __fp16 *outptr2 = outptr1 + ldout;
- __fp16 *outptr3 = outptr2 + ldout;
- __fp16 *outptr4 = outptr3 + ldout;
- __fp16 *outptr5 = outptr4 + ldout;
- __fp16 *outptr6 = outptr5 + ldout;
- __fp16 *outptr7 = outptr6 + ldout;
-
- prefetch_2x(outptr0);
- prefetch_2x(outptr1);
- prefetch_2x(outptr2);
- prefetch_2x(outptr3);
- prefetch_2x(outptr4);
- prefetch_2x(outptr5);
- prefetch_2x(outptr6);
- prefetch_2x(outptr7);
-
- for (int i=x0; i<xmax; i+=24) {
- __fp16 dummyres[24];
-
- /* Make sure we throw away results if Y isn't a multiple of 8.
- * We do this by pointing the result pointer at a dummy buffer
- * we later discard. */
- if ((y+7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- outptr1 = dummyres;
- // fall through
- case 5:
- outptr2 = dummyres;
- // fall through
- case 4:
- outptr3 = dummyres;
- // fall through
- case 3:
- outptr4 = dummyres;
- // fall through
- case 2:
- outptr5 = dummyres;
- // fall through
- case 1:
- outptr6 = dummyres;
- // fall through
- case 0:
- outptr7 = dummyres;
- break;
-
- default:
- UNREACHABLE("Impossible.");
-
- }
- }
-
- if (beta == (__fp16)0.0f) {
- /* If beta===0, don't read the output. */
-
- /* For ragged X, manually copy over the valid results. */
- if ((i+23) >= xmax) {
- for (int xi=0; xi<24; xi++) {
- if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]);
- outptr0++;
- *outptr1 = (alpha * inptr[xi + 24]);
- outptr1++;
- *outptr2 = (alpha * inptr[xi + 48]);
- outptr2++;
- *outptr3 = (alpha * inptr[xi + 72]);
- outptr3++;
- *outptr4 = (alpha * inptr[xi + 96]);
- outptr4++;
- *outptr5 = (alpha * inptr[xi + 120]);
- outptr5++;
- *outptr6 = (alpha * inptr[xi + 144]);
- outptr6++;
- *outptr7 = (alpha * inptr[xi + 168]);
- outptr7++;
- }
- }
- inptr += 192;
- } else {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
-#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
- ".arch armv8.2-a+fp16\n"
-#endif
- // Rows 0-1
- ASM_PREFETCH("[%[inptr], #768]")
- "LDP q0, q1, [%[inptr]]\n"
- "LDP q2, q3, [%[inptr], #32]\n"
- "LDP q4, q5, [%[inptr], #64]\n"
- "FMUL v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[inptr], #832]")
- "FMUL v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr0]], #32\n"
- "FMUL v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr0]], #16\n"
- "FMUL v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[inptr], #896]")
- "FMUL v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr1]], #32\n"
- "FMUL v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr1]], #16\n"
- ASM_PREFETCH("[%[inptr], #960]")
-
- // Rows 2-3
- ASM_PREFETCH("[%[inptr], #1024]")
- "LDP q0, q1, [%[inptr], #96]\n"
- "LDP q2, q3, [%[inptr], #128]\n"
- "LDP q4, q5, [%[inptr], #160]\n"
- "FMUL v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[inptr], #1088]")
- "FMUL v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr2]], #32\n"
- "FMUL v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr2]], #16\n"
- "FMUL v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr0], #80]")
- "FMUL v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr3]], #32\n"
- "FMUL v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr3]], #16\n"
- ASM_PREFETCH("[%[outptr1], #80]")
-
- // Rows 4-5
- ASM_PREFETCH("[%[outptr2], #80]")
- "LDP q0, q1, [%[inptr], #192]\n"
- "LDP q2, q3, [%[inptr], #224]\n"
- "LDP q4, q5, [%[inptr], #256]\n"
- "FMUL v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr3], #80]")
- "FMUL v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr4]], #32\n"
- "FMUL v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr4]], #16\n"
- "FMUL v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr4], #80]")
- "FMUL v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr5]], #32\n"
- "FMUL v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr5]], #16\n"
-
- // Rows 6-7
- ASM_PREFETCH("[%[outptr5], #80]")
- "LDP q0, q1, [%[inptr], #288]\n"
- "LDP q2, q3, [%[inptr], #320]\n"
- "LDP q4, q5, [%[inptr], #352]\n"
- "FMUL v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr6], #128]")
- "FMUL v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr6]], #32\n"
- "FMUL v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr6]], #16\n"
- "FMUL v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr7], #128]")
- "FMUL v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr7]], #32\n"
- "FMUL v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr7]], #16\n"
- "ADD %[inptr], %[inptr], #384\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- : [va] "w" (va), [vb] "w" (vb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21"
- );
- }
- } else {
- /* For ragged X, manually copy over the valid results. */
- if ((i+23) >= xmax) {
- for (int xi=0; xi<24; xi++) {
- if ((i+xi) < xmax) {
- *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
- outptr0++;
- *outptr1 = (alpha * inptr[xi + 24]) + (*outptr1 * beta);
- outptr1++;
- *outptr2 = (alpha * inptr[xi + 48]) + (*outptr2 * beta);
- outptr2++;
- *outptr3 = (alpha * inptr[xi + 72]) + (*outptr3 * beta);
- outptr3++;
- *outptr4 = (alpha * inptr[xi + 96]) + (*outptr4 * beta);
- outptr4++;
- *outptr5 = (alpha * inptr[xi + 120]) + (*outptr5 * beta);
- outptr5++;
- *outptr6 = (alpha * inptr[xi + 144]) + (*outptr6 * beta);
- outptr6++;
- *outptr7 = (alpha * inptr[xi + 168]) + (*outptr7 * beta);
- outptr7++;
- }
- }
- inptr += 192;
- } else {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
-#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
- ".arch armv8.2-a+fp16\n"
-#endif
- // Rows 0-1
- "LDP q16, q17, [%[outptr0]]\n"
- "FMUL v16.8h, v16.8h, %[vb].8h\n"
- "LDR q18, [%[outptr0], #32]\n"
- "FMUL v17.8h, v17.8h, %[vb].8h\n"
- "LDP q19, q20, [%[outptr1]]\n"
- "FMUL v18.8h, v18.8h, %[vb].8h\n"
- ASM_PREFETCH("[%[inptr], #768]")
- "LDR q21, [%[outptr1], #32]\n"
- "FMUL v19.8h, v19.8h, %[vb].8h\n"
- "LDP q0, q1, [%[inptr]]\n"
- "FMUL v20.8h, v20.8h, %[vb].8h\n"
- "LDP q2, q3, [%[inptr], #32]\n"
- "FMUL v21.8h, v21.8h, %[vb].8h\n"
- "LDP q4, q5, [%[inptr], #64]\n"
- "FMLA v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[inptr], #832]")
- "FMLA v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr0]], #32\n"
- "FMLA v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr0]], #16\n"
- "FMLA v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[inptr], #896]")
- "FMLA v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr1]], #32\n"
- "FMLA v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr1]], #16\n"
- ASM_PREFETCH("[%[inptr], #960]")
-
- // Rows 2-3
- "LDP q16, q17, [%[outptr2]]\n"
- "FMUL v16.8h, v16.8h, %[vb].8h\n"
- "LDR q18, [%[outptr2], #32]\n"
- "FMUL v17.8h, v17.8h, %[vb].8h\n"
- "LDP q19, q20, [%[outptr3]]\n"
- "FMUL v18.8h, v18.8h, %[vb].8h\n"
- ASM_PREFETCH("[%[inptr], #1024]")
- "LDR q21, [%[outptr3], #32]\n"
- "FMUL v19.8h, v19.8h, %[vb].8h\n"
- "LDP q0, q1, [%[inptr], #96]\n"
- "FMUL v20.8h, v20.8h, %[vb].8h\n"
- "LDP q2, q3, [%[inptr], #128]\n"
- "FMUL v21.8h, v21.8h, %[vb].8h\n"
- "LDP q4, q5, [%[inptr], #160]\n"
- "FMLA v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[inptr], #1088]")
- "FMLA v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr2]], #32\n"
- "FMLA v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr2]], #16\n"
- "FMLA v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr0], #80]")
- "FMLA v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr3]], #32\n"
- "FMLA v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr3]], #16\n"
- ASM_PREFETCH("[%[outptr1], #80]")
-
- // Rows 4-5
- "LDP q16, q17, [%[outptr4]]\n"
- "FMUL v16.8h, v16.8h, %[vb].8h\n"
- "LDR q18, [%[outptr4], #32]\n"
- "FMUL v17.8h, v17.8h, %[vb].8h\n"
- "LDP q19, q20, [%[outptr5]]\n"
- "FMUL v18.8h, v18.8h, %[vb].8h\n"
- ASM_PREFETCH("[%[outptr2], #80]")
- "LDR q21, [%[outptr5], #32]\n"
- "FMUL v19.8h, v19.8h, %[vb].8h\n"
- "LDP q0, q1, [%[inptr], #192]\n"
- "FMUL v20.8h, v20.8h, %[vb].8h\n"
- "LDP q2, q3, [%[inptr], #224]\n"
- "FMUL v21.8h, v21.8h, %[vb].8h\n"
- "LDP q4, q5, [%[inptr], #256]\n"
- "FMLA v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr3], #80]")
- "FMLA v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr4]], #32\n"
- "FMLA v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr4]], #16\n"
- "FMLA v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr4], #80]")
- "FMLA v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr5]], #32\n"
- "FMLA v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr5]], #16\n"
-
- // Rows 6-7
- "LDP q16, q17, [%[outptr6]]\n"
- "FMUL v16.8h, v16.8h, %[vb].8h\n"
- "LDR q18, [%[outptr6], #32]\n"
- "FMUL v17.8h, v17.8h, %[vb].8h\n"
- "LDP q19, q20, [%[outptr7]]\n"
- ASM_PREFETCH("[%[outptr5], #80]")
- "FMUL v18.8h, v18.8h, %[vb].8h\n"
- "LDR q21, [%[outptr7], #32]\n"
- "FMUL v19.8h, v19.8h, %[vb].8h\n"
- "LDP q0, q1, [%[inptr], #288]\n"
- "FMUL v20.8h, v20.8h, %[vb].8h\n"
- "LDP q2, q3, [%[inptr], #320]\n"
- "FMUL v21.8h, v21.8h, %[vb].8h\n"
- "LDP q4, q5, [%[inptr], #352]\n"
- "FMLA v16.8h, v0.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr6], #128]")
- "FMLA v17.8h, v1.8h, %[va].8h\n"
- "STP q16, q17, [%[outptr6]], #32\n"
- "FMLA v18.8h, v2.8h, %[va].8h\n"
- "STR q18, [%[outptr6]], #16\n"
- "FMLA v19.8h, v3.8h, %[va].8h\n"
- ASM_PREFETCH("[%[outptr7], #128]")
- "FMLA v20.8h, v4.8h, %[va].8h\n"
- "STP q19, q20, [%[outptr7]], #32\n"
- "FMLA v21.8h, v5.8h, %[va].8h\n"
- "STR q21, [%[outptr7]], #16\n"
- "ADD %[inptr], %[inptr], #384\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- : [va] "w" (va), [vb] "w" (vb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21"
- );
- }
- }
- }
- }
-}
-
-#endif // __aarch64__ && (FP16_KERNELS || __ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp
deleted file mode 100644
index 410a0a1..0000000
--- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Copyright (c) 2017-2019 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-template<>
-inline void MergeResults<12, 8, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t alpha, const int32_t beta) {
- UNUSED(alpha);
- const int32_t *inptr = in;
- prefetch_6x(inptr);
- prefetch_6x(inptr + 96);
-
- for (int y=y0; y<ymax; y+=8) {
- int32_t *outptr0 = out + (y * ldout) + x0;
- int32_t *outptr1 = outptr0 + ldout;
- int32_t *outptr2 = outptr1 + ldout;
- int32_t *outptr3 = outptr2 + ldout;
- int32_t *outptr4 = outptr3 + ldout;
- int32_t *outptr5 = outptr4 + ldout;
- int32_t *outptr6 = outptr5 + ldout;
- int32_t *outptr7 = outptr6 + ldout;
-
- prefetch_2x(outptr0);
- prefetch_2x(outptr1);
- prefetch_2x(outptr2);
- prefetch_2x(outptr3);
- prefetch_2x(outptr4);
- prefetch_2x(outptr5);
- prefetch_2x(outptr6);
- prefetch_2x(outptr7);
-
- for (int i=x0; i<xmax; i+=12) {
- int32_t dummyres[12];
-
- /* Make sure we throw away results if Y isn't a multiple of 8.
- * We do this by pointing the result pointer at a dummy buffer
- * we later discard. */
- if ((y+7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- outptr1 = dummyres;
- // fall through
- case 5:
- outptr2 = dummyres;
- // fall through
- case 4:
- outptr3 = dummyres;
- // fall through
- case 3:
- outptr4 = dummyres;
- // fall through
- case 2:
- outptr5 = dummyres;
- // fall through
- case 1:
- outptr6 = dummyres;
- // fall through
- case 0:
- outptr7 = dummyres;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- /* For ragged X, manually copy over the valid results. */
- if ((i+11) >= xmax) {
- for (int xi=0; xi<12; xi++) {
- if ((i+xi) < xmax) {
- *outptr0 = (inptr[xi]) + (*outptr0 * beta);
- outptr0++;
- *outptr1 = (inptr[xi + 12]) + (*outptr1 * beta);
- outptr1++;
- *outptr2 = (inptr[xi + 24]) + (*outptr2 * beta);
- outptr2++;
- *outptr3 = (inptr[xi + 36]) + (*outptr3 * beta);
- outptr3++;
- *outptr4 = (inptr[xi + 48]) + (*outptr4 * beta);
- outptr4++;
- *outptr5 = (inptr[xi + 60]) + (*outptr5 * beta);
- outptr5++;
- *outptr6 = (inptr[xi + 72]) + (*outptr6 * beta);
- outptr6++;
- *outptr7 = (inptr[xi + 84]) + (*outptr7 * beta);
- outptr7++;
- }
- }
- inptr += 96;
- } else {
- if (beta == 0u) {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- // Row 0
- ASM_PREFETCH("[%x[outptr1], #192]")
- "ldr q0, [%x[inptr]]\n"
- "ldr q1, [%x[inptr], #0x10]\n"
- "ldr q2, [%x[inptr], #0x20]\n"
-
- // Row 1
- ASM_PREFETCH("[%x[outptr2], #192]")
- "ldr q3, [%x[inptr], #0x30]\n"
- "str q0, [%x[outptr0]], #0x10\n"
- "ldr q4, [%x[inptr], #0x40]\n"
- "str q1, [%x[outptr0]], #0x10\n"
- "ldr q5, [%x[inptr], #0x50]\n"
- "str q2, [%x[outptr0]], #0x10\n"
-
- // Row 2
- ASM_PREFETCH("[%x[outptr3], #192]")
- "ldr q0, [%x[inptr], #0x60]\n"
- "str q3, [%x[outptr1]], #0x10\n"
- "ldr q1, [%x[inptr], #0x70]\n"
- "str q4, [%x[outptr1]], #0x10\n"
- "ldr q2, [%x[inptr], #0x80]\n"
- "str q5, [%x[outptr1]], #0x10\n"
-
- // Row 3
- ASM_PREFETCH("[%x[outptr4], #192]")
- "ldr q3, [%x[inptr], #0x90]\n"
- "str q0, [%x[outptr2]], #0x10\n"
- "ldr q4, [%x[inptr], #0xa0]\n"
- "str q1, [%x[outptr2]], #0x10\n"
- "ldr q5, [%x[inptr], #0xb0]\n"
- "str q2, [%x[outptr2]], #0x10\n"
-
- // Row 4
- ASM_PREFETCH("[%x[outptr5], #192]")
- "ldr q0, [%x[inptr], #0xc0]\n"
- "str q3, [%x[outptr3]], #0x10\n"
- "ldr q1, [%x[inptr], #0xd0]\n"
- "str q4, [%x[outptr3]], #0x10\n"
- "ldr q2, [%x[inptr], #0xe0]\n"
- "str q5, [%x[outptr3]], #0x10\n"
-
- // Row 5
- ASM_PREFETCH("[%x[outptr6], #192]")
- "ldr q3, [%x[inptr], #0xf0]\n"
- "str q0, [%x[outptr4]], #0x10\n"
- "ldr q4, [%x[inptr], #0x100]\n"
- "str q1, [%x[outptr4]], #0x10\n"
- "ldr q5, [%x[inptr], #0x110]\n"
- "str q2, [%x[outptr4]], #0x10\n"
-
- // Row 6
- ASM_PREFETCH("[%x[outptr7], #192]")
- "ldr q0, [%x[inptr], #0x120]\n"
- "str q3, [%x[outptr5]], #0x10\n"
- "ldr q1, [%x[inptr], #0x130]\n"
- "str q4, [%x[outptr5]], #0x10\n"
- "ldr q2, [%x[inptr], #0x140]\n"
- "str q5, [%x[outptr5]], #0x10\n"
-
- // Row 7
- "ldr q3, [%x[inptr], #0x150]\n"
- "str q0, [%x[outptr6]], #0x10\n"
- "ldr q4, [%x[inptr], #0x160]\n"
- "str q1, [%x[outptr6]], #0x10\n"
- "ldr q5, [%x[inptr], #0x170]\n"
- "str q2, [%x[outptr6]], #0x10\n"
- "str q3, [%x[outptr7]], #0x10\n"
- "str q4, [%x[outptr7]], #0x10\n"
- "str q5, [%x[outptr7]], #0x10\n"
-
- "add %x[inptr], %x[inptr], #0x180\n"
- : [outptr0] "+r" (outptr0),
- [outptr1] "+r" (outptr1),
- [outptr2] "+r" (outptr2),
- [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4),
- [outptr5] "+r" (outptr5),
- [outptr6] "+r" (outptr6),
- [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6"
- );
- } else {
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- // Row 0
- ASM_PREFETCH("[%x[outptr1], #192]")
- "ldr q3, [%x[outptr0]]\n"
- "ldr q4, [%x[outptr0], #0x10]\n"
- "ldr q5, [%x[outptr0], #0x20]\n"
- "ldr q6, [%x[inptr]]\n"
- "ldr q7, [%x[inptr], #0x10]\n"
- "ldr q8, [%x[inptr], #0x20]\n"
- "add v3.4s, v3.4s, v6.4s\n"
- "ldr q0, [%x[outptr1]]\n"
- "add v4.4s, v4.4s, v7.4s\n"
- "ldr q1, [%x[outptr1], #0x10]\n"
- "add v5.4s, v5.4s, v8.4s\n"
- "ldr q2, [%x[outptr1], #0x20]\n"
-
- // Row 1
- ASM_PREFETCH("[%x[outptr2], #192]")
- "ldr q6, [%x[inptr], #0x30]\n"
- "str q3, [%x[outptr0]], #0x10\n"
- "ldr q7, [%x[inptr], #0x40]\n"
- "str q4, [%x[outptr0]], #0x10\n"
- "ldr q8, [%x[inptr], #0x50]\n"
- "str q5, [%x[outptr0]], #0x10\n"
- "add v0.4s, v0.4s, v6.4s\n"
- "ldr q3, [%x[outptr2]]\n"
- "add v1.4s, v1.4s, v7.4s\n"
- "ldr q4, [%x[outptr2], #0x10]\n"
- "add v2.4s, v2.4s, v8.4s\n"
- "ldr q5, [%x[outptr2], #0x20]\n"
-
- // Row 2
- ASM_PREFETCH("[%x[outptr3], #192]")
- "ldr q6, [%x[inptr], #0x60]\n"
- "str q0, [%x[outptr1]], #0x10\n"
- "ldr q7, [%x[inptr], #0x70]\n"
- "str q1, [%x[outptr1]], #0x10\n"
- "ldr q8, [%x[inptr], #0x80]\n"
- "str q2, [%x[outptr1]], #0x10\n"
- "add v3.4s, v3.4s, v6.4s\n"
- "ldr q0, [%x[outptr3]]\n"
- "add v4.4s, v4.4s, v7.4s\n"
- "ldr q1, [%x[outptr3], #0x10]\n"
- "add v5.4s, v5.4s, v8.4s\n"
- "ldr q2, [%x[outptr3], #0x20]\n"
-
- // Row 3
- ASM_PREFETCH("[%x[outptr4], #192]")
- "ldr q6, [%x[inptr], #0x90]\n"
- "str q3, [%x[outptr2]], #0x10\n"
- "ldr q7, [%x[inptr], #0xa0]\n"
- "str q4, [%x[outptr2]], #0x10\n"
- "ldr q8, [%x[inptr], #0xb0]\n"
- "str q5, [%x[outptr2]], #0x10\n"
- "add v0.4s, v0.4s, v6.4s\n"
- "ldr q3, [%x[outptr4]]\n"
- "add v1.4s, v1.4s, v7.4s\n"
- "ldr q4, [%x[outptr4], #0x10]\n"
- "add v2.4s, v2.4s, v8.4s\n"
- "ldr q5, [%x[outptr4], #0x20]\n"
-
- // Row 4
- ASM_PREFETCH("[%x[outptr5], #192]")
- "ldr q6, [%x[inptr], #0xc0]\n"
- "str q0, [%x[outptr3]], #0x10\n"
- "ldr q7, [%x[inptr], #0xd0]\n"
- "str q1, [%x[outptr3]], #0x10\n"
- "ldr q8, [%x[inptr], #0xe0]\n"
- "str q2, [%x[outptr3]], #0x10\n"
- "add v3.4s, v3.4s, v6.4s\n"
- "ldr q0, [%x[outptr5]]\n"
- "add v4.4s, v4.4s, v7.4s\n"
- "ldr q1, [%x[outptr5], #0x10]\n"
- "add v5.4s, v5.4s, v8.4s\n"
- "ldr q2, [%x[outptr5], #0x20]\n"
-
- // Row 5
- ASM_PREFETCH("[%x[outptr6], #192]")
- "ldr q6, [%x[inptr], #0xf0]\n"
- "str q3, [%x[outptr4]], #0x10\n"
- "ldr q7, [%x[inptr], #0x100]\n"
- "str q4, [%x[outptr4]], #0x10\n"
- "ldr q8, [%x[inptr], #0x110]\n"
- "str q5, [%x[outptr4]], #0x10\n"
- "add v0.4s, v0.4s, v6.4s\n"
- "ldr q3, [%x[outptr6]]\n"
- "add v1.4s, v1.4s, v7.4s\n"
- "ldr q4, [%x[outptr6], #0x10]\n"
- "add v2.4s, v2.4s, v8.4s\n"
- "ldr q5, [%x[outptr6], #0x20]\n"
-
- // Row 6
- ASM_PREFETCH("[%x[outptr7], #192]")
- "ldr q6, [%x[inptr], #0x120]\n"
- "str q0, [%x[outptr5]], #0x10\n"
- "ldr q7, [%x[inptr], #0x130]\n"
- "str q1, [%x[outptr5]], #0x10\n"
- "ldr q8, [%x[inptr], #0x140]\n"
- "str q2, [%x[outptr5]], #0x10\n"
- "add v3.4s, v3.4s, v6.4s\n"
- "ldr q0, [%x[outptr7]]\n"
- "add v4.4s, v4.4s, v7.4s\n"
- "ldr q1, [%x[outptr7], #0x10]\n"
- "add v5.4s, v5.4s, v8.4s\n"
- "ldr q2, [%x[outptr7], #0x20]\n"
-
- // Row 7
- "ldr q6, [%x[inptr], #0x150]\n"
- "str q3, [%x[outptr6]], #0x10\n"
- "ldr q7, [%x[inptr], #0x160]\n"
- "str q4, [%x[outptr6]], #0x10\n"
- "ldr q8, [%x[inptr], #0x170]\n"
- "str q5, [%x[outptr6]], #0x10\n"
- "add v0.4s, v0.4s, v6.4s\n"
- "add v1.4s, v1.4s, v7.4s\n"
- "add v2.4s, v2.4s, v8.4s\n"
- "str q0, [%x[outptr7]], #0x10\n"
- "str q1, [%x[outptr7]], #0x10\n"
- "str q2, [%x[outptr7]], #0x10\n"
-
- "add %x[inptr], %x[inptr], #0x180\n"
- : [outptr0] "+r" (outptr0),
- [outptr1] "+r" (outptr1),
- [outptr2] "+r" (outptr2),
- [outptr3] "+r" (outptr3),
- [outptr4] "+r" (outptr4),
- [outptr5] "+r" (outptr5),
- [outptr6] "+r" (outptr6),
- [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr)
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"
- );
-
- }
- }
- }
- }
-}
-
-template<>
-inline void MergeResults<12, 8>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t alpha, const uint32_t beta) {
- // Since the above code uses only MUL and MLA instructions discard the "unsignedness" and proceed safely.
- MergeResults<12, 8>(reinterpret_cast<int32_t*>(out), reinterpret_cast<const int32_t*>(in), ldout, y0, ymax, x0, xmax, static_cast<const int32_t>(alpha), static_cast<const int32_t>(beta));
-}
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp
new file mode 100644
index 0000000..313f829
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp
@@ -0,0 +1,1595 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+template<>
+void MergeResults<12, 8, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append)
+{
+ UNUSED(act);
+
+ const int32_t *inptr = in;
+ int32_t nullbias[12] = { 0 };
+
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (12 * sizeof(int32_t)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
+ int32_t *outptr0 = out + (y * ldout) + x0;
+ int32_t *outptr1 = outptr0 + ldout;
+ int32_t *outptr2 = outptr1 + ldout;
+ int32_t *outptr3 = outptr2 + ldout;
+ int32_t *outptr4 = outptr3 + ldout;
+ int32_t *outptr5 = outptr4 + ldout;
+ int32_t *outptr6 = outptr5 + ldout;
+ int32_t *outptr7 = outptr6 + ldout;
+
+ const int height = ymax - y;
+
+ for (int i=x0; i<xmax; i+=12)
+ {
+ if (append)
+ {
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "str q13, [%[outptr1]]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr3]]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ *outptr5 += inptr[xi + 60];
+ outptr5++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "str q16, [%[outptr2]]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ *outptr5 += inptr[xi + 60];
+ outptr5++;
+ *outptr6 += inptr[xi + 72];
+ outptr6++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "str q16, [%[outptr2]]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "ldr q4, [%[outptr6]]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q12, [%[outptr6]]\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ *outptr5 += inptr[xi + 60];
+ outptr5++;
+ *outptr6 += inptr[xi + 72];
+ outptr6++;
+ *outptr7 += inptr[xi + 84];
+ outptr7++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "ldr q4, [%[outptr6]]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "ldr q7, [%[outptr7]]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q12, [%[outptr6]]\n"
+ "ldr q15, [%[inptr], #0x150]\n"
+ "ldr q8, [%[outptr7], #0x10]\n"
+ "ldr q16, [%[inptr], #0x160]\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr7], #0x20]\n"
+ "ldr q17, [%[inptr], #0x170]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q15, [%[outptr7]]\n"
+ "str q16, [%[outptr7], #0x10]\n"
+ "str q17, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ else
+ {
+ const int32_t *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "ldr q13, [%[inptr]]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "str q19, [%[outptr2]]\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "str q19, [%[outptr2]]\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "str q14, [%[outptr3]]\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "str q19, [%[outptr2]]\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "str q14, [%[outptr3]]\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "str q17, [%[outptr4]]\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ *outptr5 = biasptr[xi] + inptr[xi + 60];
+ outptr5++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "str q19, [%[outptr2]]\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v20.4s, v20.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "str q14, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v3.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "str q17, [%[outptr4]]\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "str q20, [%[outptr5]]\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ *outptr5 = biasptr[xi] + inptr[xi + 60];
+ outptr5++;
+ *outptr6 = biasptr[xi] + inptr[xi + 72];
+ outptr6++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "str q19, [%[outptr2]]\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v20.4s, v20.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "str q14, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "add v13.4s, v13.4s, v3.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v15.4s, v15.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "str q17, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v16.4s, v16.4s, v3.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "str q20, [%[outptr5]]\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "str q15, [%[outptr6]]\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ *outptr5 = biasptr[xi] + inptr[xi + 60];
+ outptr5++;
+ *outptr6 = biasptr[xi] + inptr[xi + 72];
+ outptr6++;
+ *outptr7 = biasptr[xi] + inptr[xi + 84];
+ outptr7++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "str q19, [%[outptr2]]\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v20.4s, v20.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "str q14, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "add v13.4s, v13.4s, v3.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v15.4s, v15.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "str q17, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "add v16.4s, v16.4s, v3.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "ldr q18, [%[inptr], #0x150]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "ldr q19, [%[inptr], #0x160]\n"
+ "str q20, [%[outptr5]]\n"
+ "ldr q20, [%[inptr], #0x170]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "str q15, [%[outptr6]]\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "str q18, [%[outptr7]]\n"
+ "str q19, [%[outptr7], #0x10]\n"
+ "str q20, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ }
+ }
+}
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp
new file mode 100644
index 0000000..a93060f
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp
@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+template<>
+void MergeResults<4, 4, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append)
+{
+ UNUSED(act);
+
+ const int32_t *inptr = in;
+ int32_t nullbias[4] = { 0 };
+
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (4 * sizeof(int32_t)));
+ }
+
+ for (int y=y0; y<ymax; y+=4)
+ {
+ int32_t *outptr0 = out + (y * ldout) + x0;
+ int32_t *outptr1 = outptr0 + ldout;
+ int32_t *outptr2 = outptr1 + ldout;
+ int32_t *outptr3 = outptr2 + ldout;
+
+ const int height = ymax - y;
+
+ for (int i=x0; i<xmax; i+=4)
+ {
+ if (append)
+ {
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 4];
+ outptr1++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q3, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x20]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "str q11, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 4];
+ outptr1++;
+ *outptr2 += inptr[xi + 8];
+ outptr2++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q3, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x20]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "str q12, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 4:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 4];
+ outptr1++;
+ *outptr2 += inptr[xi + 8];
+ outptr2++;
+ *outptr3 += inptr[xi + 12];
+ outptr3++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q3, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x20]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x20]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr3]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr1]]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "str q12, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q13, [%[outptr3]]\n"
+ "add %[outptr3], %[outptr3], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ else
+ {
+ const int32_t *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 4];
+ outptr1++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x20]\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v12.4s, v12.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "str q12, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 4];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 8];
+ outptr2++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x20]\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x20]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x20]\n"
+ "add v12.4s, v12.4s, v2.4s\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "str q12, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "str q13, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 4:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 4];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 8];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 12];
+ outptr3++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x20]\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x20]\n"
+ "ldr q14, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x20]\n"
+ "add v12.4s, v12.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "str q12, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "str q13, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ "str q14, [%[outptr3]]\n"
+ "add %[outptr3], %[outptr3], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ }
+ }
+}
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp
new file mode 100644
index 0000000..5569f51
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp
@@ -0,0 +1,1595 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+template<>
+void MergeResults<12, 8, false>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append)
+{
+ UNUSED(act);
+
+ const uint32_t *inptr = in;
+ uint32_t nullbias[12] = { 0 };
+
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (12 * sizeof(uint32_t)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
+ uint32_t *outptr0 = out + (y * ldout) + x0;
+ uint32_t *outptr1 = outptr0 + ldout;
+ uint32_t *outptr2 = outptr1 + ldout;
+ uint32_t *outptr3 = outptr2 + ldout;
+ uint32_t *outptr4 = outptr3 + ldout;
+ uint32_t *outptr5 = outptr4 + ldout;
+ uint32_t *outptr6 = outptr5 + ldout;
+ uint32_t *outptr7 = outptr6 + ldout;
+
+ const int height = ymax - y;
+
+ for (int i=x0; i<xmax; i+=12)
+ {
+ if (append)
+ {
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "str q13, [%[outptr1]]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr3]]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ *outptr5 += inptr[xi + 60];
+ outptr5++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "str q16, [%[outptr2]]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ *outptr5 += inptr[xi + 60];
+ outptr5++;
+ *outptr6 += inptr[xi + 72];
+ outptr6++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "str q16, [%[outptr2]]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "ldr q4, [%[outptr6]]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q12, [%[outptr6]]\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 12];
+ outptr1++;
+ *outptr2 += inptr[xi + 24];
+ outptr2++;
+ *outptr3 += inptr[xi + 36];
+ outptr3++;
+ *outptr4 += inptr[xi + 48];
+ outptr4++;
+ *outptr5 += inptr[xi + 60];
+ outptr5++;
+ *outptr6 += inptr[xi + 72];
+ outptr6++;
+ *outptr7 += inptr[xi + 84];
+ outptr7++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q3, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr0], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "ldr q6, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q12, [%[outptr0], #0x20]\n"
+ "ldr q14, [%[inptr], #0x40]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "ldr q7, [%[outptr1], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q13, [%[outptr1]]\n"
+ "ldr q15, [%[inptr], #0x50]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ldr q8, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q14, [%[outptr1], #0x10]\n"
+ "ldr q16, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q15, [%[outptr1], #0x20]\n"
+ "ldr q17, [%[inptr], #0x70]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "ldr q2, [%[outptr2], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q16, [%[outptr2]]\n"
+ "ldr q10, [%[inptr], #0x80]\n"
+ "ldr q3, [%[outptr3]]\n"
+ "ldr q11, [%[inptr], #0x90]\n"
+ "str q17, [%[outptr2], #0x10]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q4, [%[outptr3], #0x10]\n"
+ "ldr q12, [%[inptr], #0xa0]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "ldr q5, [%[outptr3], #0x20]\n"
+ "ldr q13, [%[inptr], #0xb0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr4]]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0xc0]\n"
+ "ldr q7, [%[outptr4], #0x10]\n"
+ "ldr q15, [%[inptr], #0xd0]\n"
+ "str q12, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "ldr q16, [%[inptr], #0xe0]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "str q13, [%[outptr3], #0x20]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0xf0]\n"
+ "ldr q2, [%[outptr5], #0x10]\n"
+ "ldr q10, [%[inptr], #0x100]\n"
+ "str q15, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "ldr q3, [%[outptr5], #0x20]\n"
+ "ldr q11, [%[inptr], #0x110]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q16, [%[outptr4], #0x20]\n"
+ "ldr q4, [%[outptr6]]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q17, [%[outptr5]]\n"
+ "ldr q12, [%[inptr], #0x120]\n"
+ "ldr q5, [%[outptr6], #0x10]\n"
+ "ldr q13, [%[inptr], #0x130]\n"
+ "str q10, [%[outptr5], #0x10]\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "ldr q6, [%[outptr6], #0x20]\n"
+ "ldr q14, [%[inptr], #0x140]\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "ldr q7, [%[outptr7]]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "add v14.4s, v14.4s, v6.4s\n"
+ "str q12, [%[outptr6]]\n"
+ "ldr q15, [%[inptr], #0x150]\n"
+ "ldr q8, [%[outptr7], #0x10]\n"
+ "ldr q16, [%[inptr], #0x160]\n"
+ "str q13, [%[outptr6], #0x10]\n"
+ "add v15.4s, v15.4s, v7.4s\n"
+ "ldr q9, [%[outptr7], #0x20]\n"
+ "ldr q17, [%[inptr], #0x170]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v16.4s, v16.4s, v8.4s\n"
+ "str q14, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "add v17.4s, v17.4s, v9.4s\n"
+ "str q15, [%[outptr7]]\n"
+ "str q16, [%[outptr7], #0x10]\n"
+ "str q17, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ else
+ {
+ const uint32_t *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "ldr q13, [%[inptr]]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "str q19, [%[outptr2]]\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "str q16, [%[outptr1]]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "str q19, [%[outptr2]]\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "str q14, [%[outptr3]]\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "str q19, [%[outptr2]]\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "str q14, [%[outptr3]]\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "str q17, [%[outptr4]]\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ *outptr5 = biasptr[xi] + inptr[xi + 60];
+ outptr5++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "str q19, [%[outptr2]]\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v20.4s, v20.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "str q14, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v13.4s, v13.4s, v3.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "str q17, [%[outptr4]]\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "str q20, [%[outptr5]]\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ *outptr5 = biasptr[xi] + inptr[xi + 60];
+ outptr5++;
+ *outptr6 = biasptr[xi] + inptr[xi + 72];
+ outptr6++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "str q19, [%[outptr2]]\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v20.4s, v20.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "str q14, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "add v13.4s, v13.4s, v3.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v15.4s, v15.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "str q17, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v16.4s, v16.4s, v3.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "str q20, [%[outptr5]]\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "str q15, [%[outptr6]]\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<11; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 12];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 24];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 36];
+ outptr3++;
+ *outptr4 = biasptr[xi] + inptr[xi + 48];
+ outptr4++;
+ *outptr5 = biasptr[xi] + inptr[xi + 60];
+ outptr5++;
+ *outptr6 = biasptr[xi] + inptr[xi + 72];
+ outptr6++;
+ *outptr7 = biasptr[xi] + inptr[xi + 84];
+ outptr7++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "ldr q3, [%[biasptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ldr q4, [%[biasptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ldr q13, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "ldr q14, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "ldr q15, [%[inptr], #0x20]\n"
+ "ldr q16, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add v14.4s, v14.4s, v3.4s\n"
+ "str q13, [%[outptr0]]\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0x40]\n"
+ "add v16.4s, v16.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0x50]\n"
+ "ldr q19, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add v17.4s, v17.4s, v3.4s\n"
+ "str q14, [%[outptr0], #0x10]\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "ldr q20, [%[inptr], #0x70]\n"
+ "add v19.4s, v19.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x80]\n"
+ "ldr q14, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add v20.4s, v20.4s, v3.4s\n"
+ "str q15, [%[outptr0], #0x20]\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0xa0]\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "str q16, [%[outptr1]]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "ldr q16, [%[inptr], #0xb0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "str q17, [%[outptr1], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "ldr q17, [%[inptr], #0xc0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "str q18, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add v17.4s, v17.4s, v2.4s\n"
+ "ldr q18, [%[inptr], #0xd0]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "str q19, [%[outptr2]]\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "add v18.4s, v18.4s, v3.4s\n"
+ "ldr q19, [%[inptr], #0xe0]\n"
+ "str q20, [%[outptr2], #0x10]\n"
+ "ldr q20, [%[inptr], #0xf0]\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "str q13, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add v20.4s, v20.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x100]\n"
+ "str q14, [%[outptr3]]\n"
+ "ldr q14, [%[inptr], #0x110]\n"
+ "add v13.4s, v13.4s, v3.4s\n"
+ "str q15, [%[outptr3], #0x10]\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "ldr q15, [%[inptr], #0x120]\n"
+ "str q16, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add v15.4s, v15.4s, v2.4s\n"
+ "ldr q16, [%[inptr], #0x130]\n"
+ "str q17, [%[outptr4]]\n"
+ "ldr q17, [%[inptr], #0x140]\n"
+ "add v16.4s, v16.4s, v3.4s\n"
+ "str q18, [%[outptr4], #0x10]\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "ldr q18, [%[inptr], #0x150]\n"
+ "str q19, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "ldr q19, [%[inptr], #0x160]\n"
+ "str q20, [%[outptr5]]\n"
+ "ldr q20, [%[inptr], #0x170]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "str q13, [%[outptr5], #0x10]\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "str q14, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "str q15, [%[outptr6]]\n"
+ "str q16, [%[outptr6], #0x10]\n"
+ "str q17, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "str q18, [%[outptr7]]\n"
+ "str q19, [%[outptr7], #0x10]\n"
+ "str q20, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ }
+ }
+}
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp
new file mode 100644
index 0000000..fd01bb2
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp
@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+template<>
+void MergeResults<4, 4, false>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append)
+{
+ UNUSED(act);
+
+ const uint32_t *inptr = in;
+ uint32_t nullbias[4] = { 0 };
+
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (4 * sizeof(uint32_t)));
+ }
+
+ for (int y=y0; y<ymax; y+=4)
+ {
+ uint32_t *outptr0 = out + (y * ldout) + x0;
+ uint32_t *outptr1 = outptr0 + ldout;
+ uint32_t *outptr2 = outptr1 + ldout;
+ uint32_t *outptr3 = outptr2 + ldout;
+
+ const int height = ymax - y;
+
+ for (int i=x0; i<xmax; i+=4)
+ {
+ if (append)
+ {
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 4];
+ outptr1++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q3, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x20]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "str q11, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 4];
+ outptr1++;
+ *outptr2 += inptr[xi + 8];
+ outptr2++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q3, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x20]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "str q12, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 4:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 += inptr[xi];
+ outptr0++;
+ *outptr1 += inptr[xi + 4];
+ outptr1++;
+ *outptr2 += inptr[xi + 8];
+ outptr2++;
+ *outptr3 += inptr[xi + 12];
+ outptr3++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q10, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q3, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x20]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "ldr q11, [%[inptr], #0x10]\n"
+ "ldr q4, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x20]\n"
+ "ldr q12, [%[inptr], #0x20]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x20]\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "str q10, [%[outptr0]]\n"
+ "ldr q5, [%[outptr3]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "str q11, [%[outptr1]]\n"
+ "ldr q13, [%[inptr], #0x30]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "str q12, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ "add v13.4s, v13.4s, v5.4s\n"
+ "str q13, [%[outptr3]]\n"
+ "add %[outptr3], %[outptr3], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ else
+ {
+ const uint32_t *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
+ case 1:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 4];
+ outptr1++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x20]\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v12.4s, v12.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "str q12, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 4];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 8];
+ outptr2++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x20]\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x20]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x20]\n"
+ "add v12.4s, v12.4s, v2.4s\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "str q12, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "str q13, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 4:
+ {
+ if ((i+3) >= xmax)
+ {
+ for (int xi=0; xi<3; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = biasptr[xi] + inptr[xi];
+ outptr0++;
+ *outptr1 = biasptr[xi] + inptr[xi + 4];
+ outptr1++;
+ *outptr2 = biasptr[xi] + inptr[xi + 8];
+ outptr2++;
+ *outptr3 = biasptr[xi] + inptr[xi + 12];
+ outptr3++;
+ }
+ }
+ inptr += 16;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q2, [%[biasptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x40]\n"
+ "ldr q11, [%[inptr]]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x20]\n"
+ "ldr q12, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x20]\n"
+ "add v11.4s, v11.4s, v2.4s\n"
+ "ldr q13, [%[inptr], #0x20]\n"
+ "ldr q14, [%[inptr], #0x30]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x20]\n"
+ "add v12.4s, v12.4s, v2.4s\n"
+ "str q11, [%[outptr0]]\n"
+ "add v13.4s, v13.4s, v2.4s\n"
+ "add %[outptr0], %[outptr0], #0x10\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "str q12, [%[outptr1]]\n"
+ "add %[outptr1], %[outptr1], #0x10\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x20]\n"
+ "add %[inptr], %[inptr], #0x40\n"
+ "str q13, [%[outptr2]]\n"
+ "add %[outptr2], %[outptr2], #0x10\n"
+ "str q14, [%[outptr3]]\n"
+ "add %[outptr3], %[outptr3], #0x10\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3),
+ [inptr] "+r" (inptr)
+ : [biasptr] "r" (biasptr)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ }
+ }
+}
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/merges/list.hpp b/src/core/NEON/kernels/arm_gemm/merges/list.hpp
index 788a957..4edb497 100644
--- a/src/core/NEON/kernels/arm_gemm/merges/list.hpp
+++ b/src/core/NEON/kernels/arm_gemm/merges/list.hpp
@@ -22,10 +22,13 @@
* SOFTWARE.
*/
#include "a32_merge_float_8x6.hpp"
-#include "a64_merge_float_12x8.hpp"
-#include "a64_merge_float_to_half_12x8.hpp"
-#include "a64_merge_half_24x8.hpp"
-#include "a64_merge_int32_12x8.hpp"
+#include "a64_merge_fp16_24x8.hpp"
+#include "a64_merge_fp32_12x8.hpp"
+#include "a64_merge_s32_12x8.hpp"
+#include "a64_merge_s32_4x4.hpp"
+#include "a64_merge_u32_12x8.hpp"
+#include "a64_merge_u32_4x4.hpp"
+#include "sve_merge_fp16_3VLx8.hpp"
#include "sve_merge_fp32_3VLx8.hpp"
#include "sve_merge_s32_3VLx8.hpp"
#include "sve_merge_u32_3VLx8.hpp"
diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp
new file mode 100644
index 0000000..e5efc09
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp
@@ -0,0 +1,1879 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+template<>
+void MergeResults<3, 8, true>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const __fp16 *bias, Activation act, bool append)
+{
+ const __fp16 *inptr = in;
+ __fp16 nullbias[384] = { 0 };
+ __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity());
+
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<__fp16>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (3 * get_vector_length<__fp16>() * sizeof(__fp16)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
+ __fp16 *outptr0 = out + (y * ldout) + x0;
+ __fp16 *outptr1 = outptr0 + ldout;
+ __fp16 *outptr2 = outptr1 + ldout;
+ __fp16 *outptr3 = outptr2 + ldout;
+ __fp16 *outptr4 = outptr3 + ldout;
+ __fp16 *outptr5 = outptr4 + ldout;
+ __fp16 *outptr6 = outptr5 + ldout;
+ __fp16 *outptr7 = outptr6 + ldout;
+
+ const int height = ymax - y;
+
+ for (int i=x0; i<xmax; i+=(3 * get_vector_length<__fp16>()))
+ {
+ if (append)
+ {
+ switch(height)
+ {
+ case 1:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 2:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z5.h, p0/z, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "st1h z13.h, p0, [%[outptr1]]\n"
+ "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 3:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z5.h, p0/z, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z8.h, p0/z, [%[outptr2]]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "st1h z13.h, p0, [%[outptr1]]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fmin z10.h, p2/m, z10.h, z0.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z16.h, p0, [%[outptr2]]\n"
+ "fmax z10.h, p2/m, z10.h, z1.h\n"
+ "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 4:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z5.h, p0/z, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z8.h, p0/z, [%[outptr2]]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "ld1h z3.h, p0/z, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z13.h, p0, [%[outptr1]]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.h, p2/m, z10.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z10.h, p2/m, z10.h, z1.h\n"
+ "st1h z16.h, p0, [%[outptr2]]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z11.h, p0/m, z11.h, z0.h\n"
+ "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmin z12.h, p1/m, z12.h, z0.h\n"
+ "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmax z11.h, p0/m, z11.h, z1.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "fmax z12.h, p1/m, z12.h, z1.h\n"
+ "st1h z11.h, p0, [%[outptr3]]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 5:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z5.h, p0/z, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z8.h, p0/z, [%[outptr2]]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "ld1h z3.h, p0/z, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z13.h, p0, [%[outptr1]]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "ld1h z6.h, p0/z, [%[outptr4]]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.h, p2/m, z10.h, z0.h\n"
+ "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.h, p2/m, z10.h, z1.h\n"
+ "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.h, p0/m, z11.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr2]]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.h, p1/m, z12.h, z0.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z11.h, p0/m, z11.h, z1.h\n"
+ "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z12.h, p1/m, z12.h, z1.h\n"
+ "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z11.h, p0, [%[outptr3]]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "st1h z14.h, p0, [%[outptr4]]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 6:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z5.h, p0/z, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z8.h, p0/z, [%[outptr2]]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "ld1h z3.h, p0/z, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z13.h, p0, [%[outptr1]]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "ld1h z6.h, p0/z, [%[outptr4]]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.h, p2/m, z10.h, z0.h\n"
+ "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.h, p2/m, z10.h, z1.h\n"
+ "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.h, p0/m, z11.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr2]]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.h, p1/m, z12.h, z0.h\n"
+ "ld1h z9.h, p0/z, [%[outptr5]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z11.h, p0/m, z11.h, z1.h\n"
+ "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "fmax z12.h, p1/m, z12.h, z1.h\n"
+ "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "ld1h z10.h, p1/z, [x8]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z11.h, p0, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "st1h z14.h, p0, [%[outptr4]]\n"
+ "fmin z17.h, p0/m, z17.h, z0.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z17.h, p0/m, z17.h, z1.h\n"
+ "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "fmin z10.h, p1/m, z10.h, z0.h\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmin z11.h, p2/m, z11.h, z0.h\n"
+ "st1h z17.h, p0, [%[outptr5]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax z10.h, p1/m, z10.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z11.h, p2/m, z11.h, z1.h\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 7:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z5.h, p0/z, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z8.h, p0/z, [%[outptr2]]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "ld1h z3.h, p0/z, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z13.h, p0, [%[outptr1]]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "ld1h z6.h, p0/z, [%[outptr4]]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.h, p2/m, z10.h, z0.h\n"
+ "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.h, p2/m, z10.h, z1.h\n"
+ "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.h, p0/m, z11.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr2]]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.h, p1/m, z12.h, z0.h\n"
+ "ld1h z9.h, p0/z, [%[outptr5]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z11.h, p0/m, z11.h, z1.h\n"
+ "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "fmax z12.h, p1/m, z12.h, z1.h\n"
+ "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "ld1h z10.h, p1/z, [x8]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z11.h, p0, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "ld1h z4.h, p0/z, [%[outptr6]]\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "ld1h z12.h, p0/z, [x8, #2, MUL VL]\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "ld1h z5.h, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "ld1h z13.h, p1/z, [x8, #3, MUL VL]\n"
+ "fmin z17.h, p0/m, z17.h, z0.h\n"
+ "ld1h z6.h, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "st1h z14.h, p0, [%[outptr4]]\n"
+ "fmin z10.h, p1/m, z10.h, z0.h\n"
+ "ld1h z14.h, p2/z, [x8, #4, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.h, p0/m, z17.h, z1.h\n"
+ "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "fmax z10.h, p1/m, z10.h, z1.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z11.h, p2/m, z11.h, z0.h\n"
+ "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "st1h z17.h, p0, [%[outptr5]]\n"
+ "fmax z11.h, p2/m, z11.h, z1.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z12.h, p0/m, z12.h, z0.h\n"
+ "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n"
+ "fmin z13.h, p1/m, z13.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n"
+ "fmax z12.h, p0/m, z12.h, z1.h\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmax z13.h, p1/m, z13.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z14.h, p2/m, z14.h, z0.h\n"
+ "st1h z12.h, p0, [%[outptr6]]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax z14.h, p2/m, z14.h, z1.h\n"
+ "st1h z13.h, p1, [%[outptr6], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "st1h z14.h, p2, [%[outptr6], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[outptr0]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z10.h, p0/z, [%[inptr]]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z5.h, p0/z, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.h, p0/m, z10.h, z0.h\n"
+ "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmax z10.h, p0/m, z10.h, z1.h\n"
+ "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.h, p1/m, z11.h, z0.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z8.h, p0/z, [%[outptr2]]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "st1h z10.h, p0, [%[outptr0]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.h, p1/m, z11.h, z1.h\n"
+ "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.h, p2/m, z12.h, z0.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.h, p2/m, z12.h, z1.h\n"
+ "ld1h z3.h, p0/z, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z13.h, p0, [%[outptr1]]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "ld1h z6.h, p0/z, [%[outptr4]]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.h, p2/m, z10.h, z0.h\n"
+ "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.h, p2/m, z10.h, z1.h\n"
+ "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.h, p0/m, z11.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr2]]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.h, p1/m, z12.h, z0.h\n"
+ "ld1h z9.h, p0/z, [%[outptr5]]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z11.h, p0/m, z11.h, z1.h\n"
+ "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "fmax z12.h, p1/m, z12.h, z1.h\n"
+ "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "ld1h z10.h, p1/z, [x8]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z11.h, p0, [%[outptr3]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "ld1h z4.h, p0/z, [%[outptr6]]\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "ld1h z12.h, p0/z, [x8, #2, MUL VL]\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "ld1h z5.h, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "fadd z10.h, z10.h, z2.h\n"
+ "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "ld1h z13.h, p1/z, [x8, #3, MUL VL]\n"
+ "fmin z17.h, p0/m, z17.h, z0.h\n"
+ "ld1h z6.h, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "st1h z14.h, p0, [%[outptr4]]\n"
+ "fmin z10.h, p1/m, z10.h, z0.h\n"
+ "ld1h z14.h, p2/z, [x8, #4, MUL VL]\n"
+ "fadd z11.h, z11.h, z3.h\n"
+ "ld1h z7.h, p0/z, [%[outptr7]]\n"
+ "fmax z17.h, p0/m, z17.h, z1.h\n"
+ "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z12.h, z12.h, z4.h\n"
+ "ld1h z15.h, p0/z, [x8, #5, MUL VL]\n"
+ "fmax z10.h, p1/m, z10.h, z1.h\n"
+ "ld1h z8.h, p1/z, [%[outptr7], #1, MUL VL]\n"
+ "fmin z11.h, p2/m, z11.h, z0.h\n"
+ "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "fadd z13.h, z13.h, z5.h\n"
+ "ld1h z16.h, p1/z, [x8, #6, MUL VL]\n"
+ "fmin z12.h, p0/m, z12.h, z0.h\n"
+ "ld1h z9.h, p2/z, [%[outptr7], #2, MUL VL]\n"
+ "fadd z14.h, z14.h, z6.h\n"
+ "st1h z17.h, p0, [%[outptr5]]\n"
+ "fmax z11.h, p2/m, z11.h, z1.h\n"
+ "ld1h z17.h, p2/z, [x8, #7, MUL VL]\n"
+ "fmin z13.h, p1/m, z13.h, z0.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z12.h, p0/m, z12.h, z1.h\n"
+ "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n"
+ "fmin z14.h, p2/m, z14.h, z0.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z13.h, p1/m, z13.h, z1.h\n"
+ "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n"
+ "fadd z15.h, z15.h, z7.h\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z14.h, p2/m, z14.h, z1.h\n"
+ "st1h z12.h, p0, [%[outptr6]]\n"
+ "fadd z16.h, z16.h, z8.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z15.h, p0/m, z15.h, z0.h\n"
+ "st1h z13.h, p1, [%[outptr6], #1, MUL VL]\n"
+ "fadd z17.h, z17.h, z9.h\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z16.h, p1/m, z16.h, z0.h\n"
+ "st1h z14.h, p2, [%[outptr6], #2, MUL VL]\n"
+ "fmax z15.h, p0/m, z15.h, z1.h\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmin z17.h, p2/m, z17.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax z16.h, p1/m, z16.h, z1.h\n"
+ "st1h z15.h, p0, [%[outptr7]]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z17.h, p2/m, z17.h, z1.h\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "st1h z16.h, p1, [%[outptr7], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "st1h z17.h, p2, [%[outptr7], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
+ "addvl %[outptr7], %[outptr7], #3\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+
+ }
+ }
+ else
+ {
+ const __fp16 *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
+ case 1:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 2:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "fadd z16.h, z16.h, z2.h\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fadd z17.h, z17.h, z3.h\n"
+ "fadd z18.h, z18.h, z4.h\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "fmin z18.h, p2/m, z18.h, z0.h\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z16.h, p0, [%[outptr1]]\n"
+ "fmax z18.h, p2/m, z18.h, z1.h\n"
+ "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 3:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.h, z16.h, z2.h\n"
+ "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.h, z17.h, z3.h\n"
+ "fadd z18.h, z18.h, z4.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "fmin z18.h, p2/m, z18.h, z0.h\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.h, z19.h, z2.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z16.h, p0, [%[outptr1]]\n"
+ "fmax z18.h, p2/m, z18.h, z1.h\n"
+ "fmin z19.h, p0/m, z19.h, z0.h\n"
+ "fadd z20.h, z20.h, z3.h\n"
+ "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fadd z13.h, z13.h, z4.h\n"
+ "fmax z19.h, p0/m, z19.h, z1.h\n"
+ "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fmin z20.h, p1/m, z20.h, z0.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "st1h z19.h, p0, [%[outptr2]]\n"
+ "fmax z20.h, p1/m, z20.h, z1.h\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 4:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.h, z16.h, z2.h\n"
+ "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.h, z17.h, z3.h\n"
+ "fadd z18.h, z18.h, z4.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "fmin z18.h, p2/m, z18.h, z0.h\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.h, z19.h, z2.h\n"
+ "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fadd z20.h, z20.h, z3.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "st1h z16.h, p0, [%[outptr1]]\n"
+ "fmax z18.h, p2/m, z18.h, z1.h\n"
+ "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z19.h, p0/m, z19.h, z0.h\n"
+ "fmin z20.h, p1/m, z20.h, z0.h\n"
+ "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fadd z13.h, z13.h, z4.h\n"
+ "fadd z14.h, z14.h, z2.h\n"
+ "fmax z19.h, p0/m, z19.h, z1.h\n"
+ "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fmax z20.h, p1/m, z20.h, z1.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "st1h z19.h, p0, [%[outptr2]]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "fadd z15.h, z15.h, z3.h\n"
+ "fadd z16.h, z16.h, z4.h\n"
+ "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "st1h z14.h, p0, [%[outptr3]]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 5:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.h, z16.h, z2.h\n"
+ "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.h, z17.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fadd z18.h, z18.h, z4.h\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.h, z19.h, z2.h\n"
+ "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.h, p2/m, z18.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr1]]\n"
+ "fmin z19.h, p0/m, z19.h, z0.h\n"
+ "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.h, z20.h, z3.h\n"
+ "fadd z13.h, z13.h, z4.h\n"
+ "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z18.h, p2/m, z18.h, z1.h\n"
+ "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fmax z19.h, p0/m, z19.h, z1.h\n"
+ "fmin z20.h, p1/m, z20.h, z0.h\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.h, z14.h, z2.h\n"
+ "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fadd z15.h, z15.h, z3.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z20.h, p1/m, z20.h, z1.h\n"
+ "st1h z19.h, p0, [%[outptr2]]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fadd z16.h, z16.h, z4.h\n"
+ "fadd z17.h, z17.h, z2.h\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "st1h z14.h, p0, [%[outptr3]]\n"
+ "fmin z17.h, p0/m, z17.h, z0.h\n"
+ "fadd z18.h, z18.h, z3.h\n"
+ "fadd z19.h, z19.h, z4.h\n"
+ "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "fmax z17.h, p0/m, z17.h, z1.h\n"
+ "fmin z18.h, p1/m, z18.h, z0.h\n"
+ "fmin z19.h, p2/m, z19.h, z0.h\n"
+ "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmax z18.h, p1/m, z18.h, z1.h\n"
+ "st1h z17.h, p0, [%[outptr4]]\n"
+ "fmax z19.h, p2/m, z19.h, z1.h\n"
+ "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 6:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.h, z16.h, z2.h\n"
+ "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.h, z17.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fadd z18.h, z18.h, z4.h\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.h, z19.h, z2.h\n"
+ "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.h, p2/m, z18.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr1]]\n"
+ "fmin z19.h, p0/m, z19.h, z0.h\n"
+ "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.h, z20.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmax z18.h, p2/m, z18.h, z1.h\n"
+ "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z19.h, p0/m, z19.h, z1.h\n"
+ "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fmin z20.h, p1/m, z20.h, z0.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fadd z13.h, z13.h, z4.h\n"
+ "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.h, z14.h, z2.h\n"
+ "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z20.h, p1/m, z20.h, z1.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "st1h z19.h, p0, [%[outptr2]]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fadd z15.h, z15.h, z3.h\n"
+ "fadd z16.h, z16.h, z4.h\n"
+ "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z2.h\n"
+ "ld1h z13.h, p1/z, [x8]\n"
+ "fadd z18.h, z18.h, z3.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "st1h z14.h, p0, [%[outptr3]]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n"
+ "fmin z17.h, p0/m, z17.h, z0.h\n"
+ "fmin z18.h, p1/m, z18.h, z0.h\n"
+ "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z19.h, z19.h, z4.h\n"
+ "fadd z20.h, z20.h, z2.h\n"
+ "fmax z17.h, p0/m, z17.h, z1.h\n"
+ "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z18.h, p1/m, z18.h, z1.h\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmin z19.h, p2/m, z19.h, z0.h\n"
+ "st1h z17.h, p0, [%[outptr4]]\n"
+ "fmin z20.h, p0/m, z20.h, z0.h\n"
+ "fadd z13.h, z13.h, z3.h\n"
+ "fadd z14.h, z14.h, z4.h\n"
+ "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "fmax z19.h, p2/m, z19.h, z1.h\n"
+ "fmax z20.h, p0/m, z20.h, z1.h\n"
+ "fmin z13.h, p1/m, z13.h, z0.h\n"
+ "fmin z14.h, p2/m, z14.h, z0.h\n"
+ "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "fmax z13.h, p1/m, z13.h, z1.h\n"
+ "st1h z20.h, p0, [%[outptr5]]\n"
+ "fmax z14.h, p2/m, z14.h, z1.h\n"
+ "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ case 7:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.h, z16.h, z2.h\n"
+ "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.h, z17.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fadd z18.h, z18.h, z4.h\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.h, z19.h, z2.h\n"
+ "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.h, p2/m, z18.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr1]]\n"
+ "fmin z19.h, p0/m, z19.h, z0.h\n"
+ "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.h, z20.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmax z18.h, p2/m, z18.h, z1.h\n"
+ "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z19.h, p0/m, z19.h, z1.h\n"
+ "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fmin z20.h, p1/m, z20.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fadd z13.h, z13.h, z4.h\n"
+ "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.h, z14.h, z2.h\n"
+ "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z20.h, p1/m, z20.h, z1.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "st1h z19.h, p0, [%[outptr2]]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fadd z15.h, z15.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fadd z16.h, z16.h, z4.h\n"
+ "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z2.h\n"
+ "ld1h z13.h, p1/z, [x8]\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "st1h z14.h, p0, [%[outptr3]]\n"
+ "fmin z17.h, p0/m, z17.h, z0.h\n"
+ "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n"
+ "fadd z18.h, z18.h, z3.h\n"
+ "fadd z19.h, z19.h, z4.h\n"
+ "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "ld1h z15.h, p0/z, [x8, #2, MUL VL]\n"
+ "fmax z17.h, p0/m, z17.h, z1.h\n"
+ "fmin z18.h, p1/m, z18.h, z0.h\n"
+ "fmin z19.h, p2/m, z19.h, z0.h\n"
+ "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "fadd z20.h, z20.h, z2.h\n"
+ "ld1h z16.h, p1/z, [x8, #3, MUL VL]\n"
+ "fadd z13.h, z13.h, z3.h\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmax z18.h, p1/m, z18.h, z1.h\n"
+ "st1h z17.h, p0, [%[outptr4]]\n"
+ "fmax z19.h, p2/m, z19.h, z1.h\n"
+ "ld1h z17.h, p2/z, [x8, #4, MUL VL]\n"
+ "fmin z20.h, p0/m, z20.h, z0.h\n"
+ "fmin z13.h, p1/m, z13.h, z0.h\n"
+ "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z14.h, z14.h, z4.h\n"
+ "fadd z15.h, z15.h, z2.h\n"
+ "fmax z20.h, p0/m, z20.h, z1.h\n"
+ "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "fmax z13.h, p1/m, z13.h, z1.h\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "fmin z14.h, p2/m, z14.h, z0.h\n"
+ "st1h z20.h, p0, [%[outptr5]]\n"
+ "fmin z15.h, p0/m, z15.h, z0.h\n"
+ "fadd z16.h, z16.h, z3.h\n"
+ "fadd z17.h, z17.h, z4.h\n"
+ "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n"
+ "fmax z14.h, p2/m, z14.h, z1.h\n"
+ "fmax z15.h, p0/m, z15.h, z1.h\n"
+ "fmin z16.h, p1/m, z16.h, z0.h\n"
+ "fmin z17.h, p2/m, z17.h, z0.h\n"
+ "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "fmax z16.h, p1/m, z16.h, z1.h\n"
+ "st1h z15.h, p0, [%[outptr6]]\n"
+ "fmax z17.h, p2/m, z17.h, z1.h\n"
+ "st1h z16.h, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1h z17.h, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ long w = xmax - i;
+ long p = 0;
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "mov z0.h, %h[maxval]\n"
+ "addvl x8, %[inptr], #16\n"
+ "mov z1.h, %h[minval]\n"
+ "whilelt p0.h, %[p], %[w]\n"
+ "inch %[p], all, mul #1\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1h z2.h, p0/z, [%[biasptr]]\n"
+ "whilelt p1.h, %[p], %[w]\n"
+ "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n"
+ "inch %[p], all, mul #1\n"
+ "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "ld1h z13.h, p0/z, [%[inptr]]\n"
+ "whilelt p2.h, %[p], %[w]\n"
+ "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.h, z13.h, z2.h\n"
+ "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fadd z14.h, z14.h, z3.h\n"
+ "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.h, p0/m, z13.h, z0.h\n"
+ "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.h, z15.h, z4.h\n"
+ "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.h, z16.h, z2.h\n"
+ "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.h, p1/m, z14.h, z0.h\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.h, p0/m, z13.h, z1.h\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.h, p1/m, z14.h, z1.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.h, p2/m, z15.h, z0.h\n"
+ "st1h z13.h, p0, [%[outptr0]]\n"
+ "fmin z16.h, p0/m, z16.h, z0.h\n"
+ "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.h, z17.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.h, p2/m, z15.h, z1.h\n"
+ "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.h, p0/m, z16.h, z1.h\n"
+ "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.h, p1/m, z17.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fadd z18.h, z18.h, z4.h\n"
+ "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.h, z19.h, z2.h\n"
+ "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.h, p1/m, z17.h, z1.h\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.h, p2/m, z18.h, z0.h\n"
+ "st1h z16.h, p0, [%[outptr1]]\n"
+ "fmin z19.h, p0/m, z19.h, z0.h\n"
+ "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.h, z20.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmax z18.h, p2/m, z18.h, z1.h\n"
+ "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z19.h, p0/m, z19.h, z1.h\n"
+ "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n"
+ "fmin z20.h, p1/m, z20.h, z0.h\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fadd z13.h, z13.h, z4.h\n"
+ "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.h, z14.h, z2.h\n"
+ "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z20.h, p1/m, z20.h, z1.h\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.h, p2/m, z13.h, z0.h\n"
+ "st1h z19.h, p0, [%[outptr2]]\n"
+ "fmin z14.h, p0/m, z14.h, z0.h\n"
+ "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n"
+ "fadd z15.h, z15.h, z3.h\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fmax z13.h, p2/m, z13.h, z1.h\n"
+ "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z14.h, p0/m, z14.h, z1.h\n"
+ "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z15.h, p1/m, z15.h, z0.h\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "fadd z16.h, z16.h, z4.h\n"
+ "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z17.h, z17.h, z2.h\n"
+ "ld1h z13.h, p1/z, [x8]\n"
+ "fmax z15.h, p1/m, z15.h, z1.h\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z16.h, p2/m, z16.h, z0.h\n"
+ "st1h z14.h, p0, [%[outptr3]]\n"
+ "fmin z17.h, p0/m, z17.h, z0.h\n"
+ "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n"
+ "fadd z18.h, z18.h, z3.h\n"
+ "addvl %[inptr], %[inptr], #24\n"
+ "fmax z16.h, p2/m, z16.h, z1.h\n"
+ "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmax z17.h, p0/m, z17.h, z1.h\n"
+ "ld1h z15.h, p0/z, [x8, #2, MUL VL]\n"
+ "fmin z18.h, p1/m, z18.h, z0.h\n"
+ "fadd z19.h, z19.h, z4.h\n"
+ "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n"
+ "fadd z20.h, z20.h, z2.h\n"
+ "ld1h z16.h, p1/z, [x8, #3, MUL VL]\n"
+ "fadd z13.h, z13.h, z3.h\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmax z18.h, p1/m, z18.h, z1.h\n"
+ "st1h z17.h, p0, [%[outptr4]]\n"
+ "fmin z19.h, p2/m, z19.h, z0.h\n"
+ "ld1h z17.h, p2/z, [x8, #4, MUL VL]\n"
+ "fmin z20.h, p0/m, z20.h, z0.h\n"
+ "fmin z13.h, p1/m, z13.h, z0.h\n"
+ "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z14.h, z14.h, z4.h\n"
+ "ld1h z18.h, p0/z, [x8, #5, MUL VL]\n"
+ "fmax z19.h, p2/m, z19.h, z1.h\n"
+ "fmax z20.h, p0/m, z20.h, z1.h\n"
+ "fmax z13.h, p1/m, z13.h, z1.h\n"
+ "fmin z14.h, p2/m, z14.h, z0.h\n"
+ "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n"
+ "fadd z15.h, z15.h, z2.h\n"
+ "ld1h z19.h, p1/z, [x8, #6, MUL VL]\n"
+ "fadd z16.h, z16.h, z3.h\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "fmax z14.h, p2/m, z14.h, z1.h\n"
+ "st1h z20.h, p0, [%[outptr5]]\n"
+ "fmin z15.h, p0/m, z15.h, z0.h\n"
+ "ld1h z20.h, p2/z, [x8, #7, MUL VL]\n"
+ "fmin z16.h, p1/m, z16.h, z0.h\n"
+ "fadd z17.h, z17.h, z4.h\n"
+ "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n"
+ "fadd z18.h, z18.h, z2.h\n"
+ "fmax z15.h, p0/m, z15.h, z1.h\n"
+ "fmax z16.h, p1/m, z16.h, z1.h\n"
+ "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n"
+ "fmin z17.h, p2/m, z17.h, z0.h\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "fmin z18.h, p0/m, z18.h, z0.h\n"
+ "st1h z15.h, p0, [%[outptr6]]\n"
+ "fadd z19.h, z19.h, z3.h\n"
+ "fmax z17.h, p2/m, z17.h, z1.h\n"
+ "fadd z20.h, z20.h, z4.h\n"
+ "st1h z16.h, p1, [%[outptr6], #1, MUL VL]\n"
+ "fmax z18.h, p0/m, z18.h, z1.h\n"
+ "fmin z19.h, p1/m, z19.h, z0.h\n"
+ "fmin z20.h, p2/m, z20.h, z0.h\n"
+ "st1h z17.h, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "fmax z19.h, p1/m, z19.h, z1.h\n"
+ "fmax z20.h, p2/m, z20.h, z1.h\n"
+ "st1h z18.h, p0, [%[outptr7]]\n"
+ "st1h z19.h, p1, [%[outptr7], #1, MUL VL]\n"
+ "st1h z20.h, p2, [%[outptr7], #2, MUL VL]\n"
+ "addvl %[outptr7], %[outptr7], #3\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr), [p] "+r" (p)
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
+ );
+ }
+ break;
+
+
+ }
+ }
+ }
+ }
+}
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp
deleted file mode 100644
index 7479c8d..0000000
--- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp
+++ /dev/null
@@ -1,1208 +0,0 @@
-/*
- * Copyright (c) 2018 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-template<>
-inline void MergeResults<2, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta)
-{
- const float *inptr = in;
-
- for (int y=y0; y<ymax; y+=8) {
- float *outptr0 = out + (y * ldout) + x0;
- float *outptr1 = outptr0 + ldout;
- float *outptr2 = outptr1 + ldout;
- float *outptr3 = outptr2 + ldout;
- float *outptr4 = outptr3 + ldout;
- float *outptr5 = outptr4 + ldout;
- float *outptr6 = outptr5 + ldout;
- float *outptr7 = outptr6 + ldout;
-
- const int height = ymax - y;
-
- for (int i=x0; i<xmax; i+=(2 * get_vector_length<float>())) {
- if (beta==0.0f)
- {
- switch(height) {
- case 1:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 2:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x40]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 3:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x40]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 4:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x40]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x40]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 5:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #-7, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x40]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 6:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-7, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x40]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x40]\n"
- "addvl %[outptr5], %[outptr5], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 7:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x40]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x40]\n"
- "addvl %[outptr5], %[outptr5], #2\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x40]\n"
- "addvl %[outptr6], %[outptr6], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- default:
- case 8:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "ld1w z7.s, p0/z, [x8, #-2, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-7, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x40]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x40]\n"
- "addvl %[outptr5], %[outptr5], #2\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x40]\n"
- "addvl %[outptr6], %[outptr6], #2\n"
- "prfm PSTL1KEEP, [%[outptr7], #0x40]\n"
- "addvl %[outptr7], %[outptr7], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
-
- }
- }
- else
- {
- switch(height) {
- case 1:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 2:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x40]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 3:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x40]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 4:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x40]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x40]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 5:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x40]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 6:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z9.s, p0/z, [%[outptr5]]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x40]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x40]\n"
- "addvl %[outptr5], %[outptr5], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- case 7:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z9.s, p0/z, [%[outptr5]]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z10.s, p0/z, [%[outptr6]]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x40]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x40]\n"
- "addvl %[outptr5], %[outptr5], #2\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x40]\n"
- "addvl %[outptr6], %[outptr6], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
- default:
- case 8:
- {
- long w = xmax - i;
- long p = 0;
- /* Optimized routine to copy an entire block */
- __asm __volatile (
- "mov z2.s, %s[alpha]\n"
- "addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
- "incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x100]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x140]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z9.s, p0/z, [%[outptr5]]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z10.s, p0/z, [%[outptr6]]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "ld1w z11.s, p0/z, [%[outptr7]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-2, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x40]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #2\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x40]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #2\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x40]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #2\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x40]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr7], #1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #2\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-1, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x40]\n"
- "addvl %[outptr4], %[outptr4], #2\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x40]\n"
- "addvl %[outptr5], %[outptr5], #2\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x40]\n"
- "addvl %[outptr6], %[outptr6], #2\n"
- "prfm PLDL1KEEP, [%[outptr7], #0x40]\n"
- "addvl %[outptr7], %[outptr7], #2\n"
- "1:\n"
- "addvl %[inptr], %[inptr], #16\n"
- : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
- [inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
- );
- }
- break;
-
-
- }
- }
- }
- }
-}
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp
index 4b51066..f7feec4 100644
--- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp
@@ -26,11 +26,33 @@
#ifdef __ARM_FEATURE_SVE
template<>
-inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta)
+void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append)
{
const float *inptr = in;
+ float nullbias[192] = { 0 };
+ float minval = - std::numeric_limits<float>::infinity();
+ float maxval = std::numeric_limits<float>::infinity();
- for (int y=y0; y<ymax; y+=8) {
+ switch(act.type)
+ {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ minval = 0.0f;
+ break;
+ }
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (3 * get_vector_length<float>() * sizeof(float)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
float *outptr0 = out + (y * ldout) + x0;
float *outptr1 = outptr0 + ldout;
float *outptr2 = outptr1 + ldout;
@@ -42,45 +64,52 @@
const int height = ymax - y;
- for (int i=x0; i<xmax; i+=(3 * get_vector_length<float>())) {
- if (beta==0.0f)
+ for (int i=x0; i<xmax; i+=(3 * get_vector_length<float>()))
+ {
+ if (append)
{
- switch(height) {
+ switch(height)
+ {
case 1:
{
long w = xmax - i;
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "1:\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -91,47 +120,61 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "1:\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -142,59 +185,82 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "1:\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fmin z10.s, p2/m, z10.s, z0.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "fmax z10.s, p2/m, z10.s, z1.s\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -205,70 +271,102 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "1:\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.s, p2/m, z10.s, z0.s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z10.s, p2/m, z10.s, z1.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z11.s, p0/m, z11.s, z0.s\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmin z12.s, p1/m, z12.s, z0.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmax z11.s, p0/m, z11.s, z1.s\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "fmax z12.s, p1/m, z12.s, z1.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -279,82 +377,123 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.s, p2/m, z10.s, z0.s\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.s, p2/m, z10.s, z1.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.s, p0/m, z11.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.s, p1/m, z12.s, z0.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z11.s, p0/m, z11.s, z1.s\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z12.s, p1/m, z12.s, z1.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "fadd z15.s, z15.s, z7.s\n"
"addvl %[outptr2], %[outptr2], #3\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
"addvl %[outptr3], %[outptr3], #3\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "1:\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -365,94 +504,144 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.s, p2/m, z10.s, z0.s\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.s, p2/m, z10.s, z1.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.s, p0/m, z11.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.s, p1/m, z12.s, z0.s\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z11.s, p0/m, z11.s, z1.s\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "fmax z12.s, p1/m, z12.s, z1.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "fmin z17.s, p0/m, z17.s, z0.s\n"
"addvl %[outptr2], %[outptr2], #3\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z17.s, p0/m, z17.s, z1.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "fmin z10.s, p1/m, z10.s, z0.s\n"
"addvl %[outptr3], %[outptr3], #3\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmin z11.s, p2/m, z11.s, z0.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax z10.s, p1/m, z10.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z11.s, p2/m, z11.s, z1.s\n"
"addvl %[outptr4], %[outptr4], #3\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "1:\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -463,106 +652,165 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #3, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #4, MUL VL]\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.s, p2/m, z10.s, z0.s\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.s, p2/m, z10.s, z1.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.s, p0/m, z11.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.s, p1/m, z12.s, z0.s\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z11.s, p0/m, z11.s, z1.s\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "fmax z12.s, p1/m, z12.s, z1.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "ld1w z4.s, p0/z, [%[outptr6]]\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
+ "fmin z17.s, p0/m, z17.s, z0.s\n"
+ "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "fmin z10.s, p1/m, z10.s, z0.s\n"
+ "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.s, p0/m, z17.s, z1.s\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "fmax z10.s, p1/m, z10.s, z1.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z11.s, p2/m, z11.s, z0.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "fmax z11.s, p2/m, z11.s, z1.s\n"
"addvl %[outptr2], %[outptr2], #3\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z12.s, p0/m, z12.s, z0.s\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "fmin z13.s, p1/m, z13.s, z0.s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "fmax z12.s, p0/m, z12.s, z1.s\n"
"addvl %[outptr3], %[outptr3], #3\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z13.s, p1/m, z13.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z14.s, p2/m, z14.s, z0.s\n"
+ "st1w z12.s, p0, [%[outptr6]]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
"addvl %[outptr4], %[outptr4], #3\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmax z14.s, p2/m, z14.s, z1.s\n"
+ "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
"addvl %[outptr5], %[outptr5], #3\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
"addvl %[outptr6], %[outptr6], #3\n"
- "1:\n"
"addvl %[inptr], %[inptr], #24\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -574,117 +822,185 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "ld1w z7.s, p0/z, [x8, #5, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #3, MUL VL]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #6, MUL VL]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmin z10.s, p0/m, z10.s, z0.s\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z8.s, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "fmul z9.s, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "ld1w z6.s, p0/z, [x8, #4, MUL VL]\n"
+ "fmax z10.s, p0/m, z10.s, z1.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "fmin z11.s, p1/m, z11.s, z0.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fmax z11.s, p1/m, z11.s, z1.s\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "fmin z12.s, p2/m, z12.s, z0.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fmax z12.s, p2/m, z12.s, z1.s\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmin z10.s, p2/m, z10.s, z0.s\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z10.s, p2/m, z10.s, z1.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "fmin z11.s, p0/m, z11.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z12.s, p1/m, z12.s, z0.s\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z11.s, p0/m, z11.s, z1.s\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "fmax z12.s, p1/m, z12.s, z1.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "ld1w z4.s, p0/z, [%[outptr6]]\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "fadd z10.s, z10.s, z2.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
+ "fmin z17.s, p0/m, z17.s, z0.s\n"
+ "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "fmin z10.s, p1/m, z10.s, z0.s\n"
+ "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
+ "fadd z11.s, z11.s, z3.s\n"
+ "ld1w z7.s, p0/z, [%[outptr7]]\n"
+ "fmax z17.s, p0/m, z17.s, z1.s\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z12.s, z12.s, z4.s\n"
+ "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n"
+ "fmax z10.s, p1/m, z10.s, z1.s\n"
+ "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n"
+ "fmin z11.s, p2/m, z11.s, z0.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "fadd z13.s, z13.s, z5.s\n"
+ "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n"
+ "fmin z12.s, p0/m, z12.s, z0.s\n"
+ "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n"
+ "fadd z14.s, z14.s, z6.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "fmax z11.s, p2/m, z11.s, z1.s\n"
+ "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n"
+ "fmin z13.s, p1/m, z13.s, z0.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z12.s, p0/m, z12.s, z1.s\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "fmin z14.s, p2/m, z14.s, z0.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z13.s, p1/m, z13.s, z1.s\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "fadd z15.s, z15.s, z7.s\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z14.s, p2/m, z14.s, z1.s\n"
+ "st1w z12.s, p0, [%[outptr6]]\n"
+ "fadd z16.s, z16.s, z8.s\n"
"addvl %[outptr2], %[outptr2], #3\n"
- "fmul z10.s, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [x8, #7, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "fmul z11.s, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n"
+ "fmin z15.s, p0/m, z15.s, z0.s\n"
+ "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "fadd z17.s, z17.s, z9.s\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmin z16.s, p1/m, z16.s, z0.s\n"
+ "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "fmax z15.s, p0/m, z15.s, z1.s\n"
"addvl %[outptr3], %[outptr3], #3\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmin z17.s, p2/m, z17.s, z0.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmax z16.s, p1/m, z16.s, z1.s\n"
+ "st1w z15.s, p0, [%[outptr7]]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z17.s, p2/m, z17.s, z1.s\n"
"addvl %[outptr4], %[outptr4], #3\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
"addvl %[outptr5], %[outptr5], #3\n"
+ "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
"addvl %[outptr6], %[outptr6], #3\n"
- "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
"addvl %[outptr7], %[outptr7], #3\n"
- "1:\n"
"addvl %[inptr], %[inptr], #24\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -694,48 +1010,54 @@
}
else
{
- switch(height) {
+ const float *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
case 1:
{
long w = xmax - i;
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "1:\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -746,59 +1068,58 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "1:\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "fadd z16.s, z16.s, z2.s\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fadd z17.s, z17.s, z3.s\n"
+ "fadd z18.s, z18.s, z4.s\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "fmin z18.s, p2/m, z18.s, z0.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "fmax z18.s, p2/m, z18.s, z1.s\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -809,77 +1130,76 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "1:\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.s, z16.s, z2.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.s, z17.s, z3.s\n"
+ "fadd z18.s, z18.s, z4.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "fmin z18.s, p2/m, z18.s, z0.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.s, z19.s, z2.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "fmax z18.s, p2/m, z18.s, z1.s\n"
+ "fmin z19.s, p0/m, z19.s, z0.s\n"
+ "fadd z20.s, z20.s, z3.s\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fadd z13.s, z13.s, z4.s\n"
+ "fmax z19.s, p0/m, z19.s, z1.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fmin z20.s, p1/m, z20.s, z0.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "fmax z20.s, p1/m, z20.s, z1.s\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -890,94 +1210,93 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "1:\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.s, z16.s, z2.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.s, z17.s, z3.s\n"
+ "fadd z18.s, z18.s, z4.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "fmin z18.s, p2/m, z18.s, z0.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.s, z19.s, z2.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fadd z20.s, z20.s, z3.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "fmax z18.s, p2/m, z18.s, z1.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fmin z19.s, p0/m, z19.s, z0.s\n"
+ "fmin z20.s, p1/m, z20.s, z0.s\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fadd z13.s, z13.s, z4.s\n"
+ "fadd z14.s, z14.s, z2.s\n"
+ "fmax z19.s, p0/m, z19.s, z1.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fmax z20.s, p1/m, z20.s, z1.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "fadd z15.s, z15.s, z3.s\n"
+ "fadd z16.s, z16.s, z4.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -988,112 +1307,111 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "1:\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.s, z16.s, z2.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.s, z17.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fadd z18.s, z18.s, z4.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.s, z19.s, z2.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.s, p2/m, z18.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "fmin z19.s, p0/m, z19.s, z0.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.s, z20.s, z3.s\n"
+ "fadd z13.s, z13.s, z4.s\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z18.s, p2/m, z18.s, z1.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fmax z19.s, p0/m, z19.s, z1.s\n"
+ "fmin z20.s, p1/m, z20.s, z0.s\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.s, z14.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fadd z15.s, z15.s, z3.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmax z20.s, p1/m, z20.s, z1.s\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fadd z16.s, z16.s, z4.s\n"
+ "fadd z17.s, z17.s, z2.s\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "fmin z17.s, p0/m, z17.s, z0.s\n"
+ "fadd z18.s, z18.s, z3.s\n"
+ "fadd z19.s, z19.s, z4.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "fmax z17.s, p0/m, z17.s, z1.s\n"
+ "fmin z18.s, p1/m, z18.s, z0.s\n"
+ "fmin z19.s, p2/m, z19.s, z0.s\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmax z18.s, p1/m, z18.s, z1.s\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "fmax z19.s, p2/m, z19.s, z1.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -1104,130 +1422,129 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z9.s, p0/z, [%[outptr5]]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "1:\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.s, z16.s, z2.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.s, z17.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fadd z18.s, z18.s, z4.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.s, z19.s, z2.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.s, p2/m, z18.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "fmin z19.s, p0/m, z19.s, z0.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.s, z20.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmax z18.s, p2/m, z18.s, z1.s\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z19.s, p0/m, z19.s, z1.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fmin z20.s, p1/m, z20.s, z0.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fadd z13.s, z13.s, z4.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.s, z14.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z20.s, p1/m, z20.s, z1.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fadd z15.s, z15.s, z3.s\n"
+ "fadd z16.s, z16.s, z4.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "fadd z18.s, z18.s, z3.s\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "fmin z17.s, p0/m, z17.s, z0.s\n"
+ "fmin z18.s, p1/m, z18.s, z0.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fadd z19.s, z19.s, z4.s\n"
+ "fadd z20.s, z20.s, z2.s\n"
+ "fmax z17.s, p0/m, z17.s, z1.s\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "fmax z18.s, p1/m, z18.s, z1.s\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmin z19.s, p2/m, z19.s, z0.s\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "fmin z20.s, p0/m, z20.s, z0.s\n"
+ "fadd z13.s, z13.s, z3.s\n"
+ "fadd z14.s, z14.s, z4.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "fmax z19.s, p2/m, z19.s, z1.s\n"
+ "fmax z20.s, p0/m, z20.s, z1.s\n"
+ "fmin z13.s, p1/m, z13.s, z0.s\n"
+ "fmin z14.s, p2/m, z14.s, z0.s\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "fmax z13.s, p1/m, z13.s, z1.s\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "fmax z14.s, p2/m, z14.s, z1.s\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -1238,148 +1555,147 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z9.s, p0/z, [%[outptr5]]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z10.s, p0/z, [%[outptr6]]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #1, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #4, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.s, z16.s, z2.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.s, z17.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fadd z18.s, z18.s, z4.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.s, z19.s, z2.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.s, p2/m, z18.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "fmin z19.s, p0/m, z19.s, z0.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.s, z20.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmax z18.s, p2/m, z18.s, z1.s\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z19.s, p0/m, z19.s, z1.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fmin z20.s, p1/m, z20.s, z0.s\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "1:\n"
+ "fadd z13.s, z13.s, z4.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.s, z14.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z20.s, p1/m, z20.s, z1.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fadd z15.s, z15.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fadd z16.s, z16.s, z4.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "fmin z17.s, p0/m, z17.s, z0.s\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "fadd z18.s, z18.s, z3.s\n"
+ "fadd z19.s, z19.s, z4.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
+ "fmax z17.s, p0/m, z17.s, z1.s\n"
+ "fmin z18.s, p1/m, z18.s, z0.s\n"
+ "fmin z19.s, p2/m, z19.s, z0.s\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "fadd z20.s, z20.s, z2.s\n"
+ "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
+ "fadd z13.s, z13.s, z3.s\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmax z18.s, p1/m, z18.s, z1.s\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "fmax z19.s, p2/m, z19.s, z1.s\n"
+ "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
+ "fmin z20.s, p0/m, z20.s, z0.s\n"
+ "fmin z13.s, p1/m, z13.s, z0.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z14.s, z14.s, z4.s\n"
+ "fadd z15.s, z15.s, z2.s\n"
+ "fmax z20.s, p0/m, z20.s, z1.s\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "fmax z13.s, p1/m, z13.s, z1.s\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "fmin z14.s, p2/m, z14.s, z0.s\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "fmin z15.s, p0/m, z15.s, z0.s\n"
+ "fadd z16.s, z16.s, z3.s\n"
+ "fadd z17.s, z17.s, z4.s\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "fmax z14.s, p2/m, z14.s, z1.s\n"
+ "fmax z15.s, p0/m, z15.s, z1.s\n"
+ "fmin z16.s, p1/m, z16.s, z0.s\n"
+ "fmin z17.s, p2/m, z17.s, z0.s\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "fmax z16.s, p1/m, z16.s, z1.s\n"
+ "st1w z15.s, p0, [%[outptr6]]\n"
+ "fmax z17.s, p2/m, z17.s, z1.s\n"
+ "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -1391,165 +1707,164 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
+ "mov z0.s, %s[maxval]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
+ "mov z1.s, %s[minval]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0]]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr]]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z9.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z10.s, p0/z, [%[outptr2]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z11.s, p0/z, [%[outptr3]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z8.s, p0/z, [%[outptr4]]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "ld1w z9.s, p0/z, [%[outptr5]]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "ld1w z10.s, p0/z, [%[outptr6]]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "ld1w z11.s, p0/z, [%[outptr7]]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-3, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #3, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr7], #1, MUL VL]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #6, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fadd z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "fmul z8.s, z8.s, z3.s\n"
- "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n"
- "fmla z8.s, p0/m, z4.s, z2.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "fmul z9.s, z9.s, z3.s\n"
- "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n"
- "fmla z9.s, p0/m, z5.s, z2.s\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "fmul z10.s, z10.s, z3.s\n"
- "ld1w z6.s, p0/z, [x8, #4, MUL VL]\n"
- "fmla z10.s, p0/m, z6.s, z2.s\n"
- "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[outptr7], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "fmul z11.s, z11.s, z3.s\n"
- "ld1w z7.s, p0/z, [x8, #7, MUL VL]\n"
- "fmla z11.s, p0/m, z7.s, z2.s\n"
- "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
+ "fadd z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "fmin z13.s, p0/m, z13.s, z0.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "fadd z15.s, z15.s, z4.s\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "fadd z16.s, z16.s, z2.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "fmin z14.s, p1/m, z14.s, z0.s\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmax z13.s, p0/m, z13.s, z1.s\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmax z14.s, p1/m, z14.s, z1.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmin z15.s, p2/m, z15.s, z0.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "fmin z16.s, p0/m, z16.s, z0.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "fadd z17.s, z17.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmax z15.s, p2/m, z15.s, z1.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "fmax z16.s, p0/m, z16.s, z1.s\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "fmin z17.s, p1/m, z17.s, z0.s\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fadd z18.s, z18.s, z4.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "fadd z19.s, z19.s, z2.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "fmax z17.s, p1/m, z17.s, z1.s\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "fmin z18.s, p2/m, z18.s, z0.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "fmin z19.s, p0/m, z19.s, z0.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "fadd z20.s, z20.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmax z18.s, p2/m, z18.s, z1.s\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "fmax z19.s, p0/m, z19.s, z1.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "fmin z20.s, p1/m, z20.s, z0.s\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
- "addvl %[outptr7], %[outptr7], #3\n"
- "1:\n"
+ "fadd z13.s, z13.s, z4.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "fadd z14.s, z14.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
+ "fmax z20.s, p1/m, z20.s, z1.s\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "fmin z13.s, p2/m, z13.s, z0.s\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "fmin z14.s, p0/m, z14.s, z0.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "fadd z15.s, z15.s, z3.s\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fmax z13.s, p2/m, z13.s, z1.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "fmax z14.s, p0/m, z14.s, z1.s\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "fmin z15.s, p1/m, z15.s, z0.s\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "fadd z16.s, z16.s, z4.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "fadd z17.s, z17.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "fmax z15.s, p1/m, z15.s, z1.s\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "fmin z16.s, p2/m, z16.s, z0.s\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "fmin z17.s, p0/m, z17.s, z0.s\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "fadd z18.s, z18.s, z3.s\n"
"addvl %[inptr], %[inptr], #24\n"
+ "fmax z16.s, p2/m, z16.s, z1.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "fmax z17.s, p0/m, z17.s, z1.s\n"
+ "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
+ "fmin z18.s, p1/m, z18.s, z0.s\n"
+ "fadd z19.s, z19.s, z4.s\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "fadd z20.s, z20.s, z2.s\n"
+ "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
+ "fadd z13.s, z13.s, z3.s\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "fmax z18.s, p1/m, z18.s, z1.s\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "fmin z19.s, p2/m, z19.s, z0.s\n"
+ "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
+ "fmin z20.s, p0/m, z20.s, z0.s\n"
+ "fmin z13.s, p1/m, z13.s, z0.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "fadd z14.s, z14.s, z4.s\n"
+ "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n"
+ "fmax z19.s, p2/m, z19.s, z1.s\n"
+ "fmax z20.s, p0/m, z20.s, z1.s\n"
+ "fmax z13.s, p1/m, z13.s, z1.s\n"
+ "fmin z14.s, p2/m, z14.s, z0.s\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "fadd z15.s, z15.s, z2.s\n"
+ "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n"
+ "fadd z16.s, z16.s, z3.s\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "fmax z14.s, p2/m, z14.s, z1.s\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "fmin z15.s, p0/m, z15.s, z0.s\n"
+ "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n"
+ "fmin z16.s, p1/m, z16.s, z0.s\n"
+ "fadd z17.s, z17.s, z4.s\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "fadd z18.s, z18.s, z2.s\n"
+ "fmax z15.s, p0/m, z15.s, z1.s\n"
+ "fmax z16.s, p1/m, z16.s, z1.s\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "fmin z17.s, p2/m, z17.s, z0.s\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "fmin z18.s, p0/m, z18.s, z0.s\n"
+ "st1w z15.s, p0, [%[outptr6]]\n"
+ "fadd z19.s, z19.s, z3.s\n"
+ "fmax z17.s, p2/m, z17.s, z1.s\n"
+ "fadd z20.s, z20.s, z4.s\n"
+ "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "fmax z18.s, p0/m, z18.s, z1.s\n"
+ "fmin z19.s, p1/m, z19.s, z0.s\n"
+ "fmin z20.s, p2/m, z20.s, z0.s\n"
+ "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "fmax z19.s, p1/m, z19.s, z1.s\n"
+ "fmax z20.s, p2/m, z20.s, z1.s\n"
+ "st1w z18.s, p0, [%[outptr7]]\n"
+ "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n"
+ "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n"
+ "addvl %[outptr7], %[outptr7], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp
index 2ea38a7..03a2824 100644
--- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp
@@ -26,11 +26,20 @@
#ifdef __ARM_FEATURE_SVE
template<>
-inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t alpha, const int32_t beta)
+void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append)
{
- const int32_t *inptr = in;
+ UNUSED(act);
- for (int y=y0; y<ymax; y+=8) {
+ const int32_t *inptr = in;
+ int32_t nullbias[192] = { 0 };
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (3 * get_vector_length<int32_t>() * sizeof(int32_t)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
int32_t *outptr0 = out + (y * ldout) + x0;
int32_t *outptr1 = outptr0 + ldout;
int32_t *outptr2 = outptr1 + ldout;
@@ -42,42 +51,44 @@
const int height = ymax - y;
- for (int i=x0; i<xmax; i+=(3 * get_vector_length<int32_t>())) {
- if (beta==0)
+ for (int i=x0; i<xmax; i+=(3 * get_vector_length<int32_t>()))
+ {
+ if (append)
{
- switch(height) {
+ switch(height)
+ {
case 1:
{
long w = xmax - i;
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "1:\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -88,41 +99,47 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "1:\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -133,50 +150,62 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -187,58 +216,76 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -249,67 +296,91 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -320,76 +391,106 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8]\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -400,85 +501,121 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
- "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "1:\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[outptr6]]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
+ "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr6]]\n"
+ "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -490,93 +627,135 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
- "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n"
- "addvl %[outptr7], %[outptr7], #3\n"
- "1:\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[outptr6]]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
+ "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z7.s, p0/z, [%[outptr7]]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr6]]\n"
+ "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n"
+ "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n"
+ "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n"
+ "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n"
+ "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "add z17.s, z17.s, z9.s\n"
+ "st1w z15.s, p0, [%[outptr7]]\n"
+ "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n"
+ "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n"
+ "addvl %[outptr7], %[outptr7], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -586,45 +765,46 @@
}
else
{
- switch(height) {
+ const int32_t *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
case 1:
{
long w = xmax - i;
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "1:\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "add z14.s, z14.s, z3.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -635,53 +815,44 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "1:\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z15.s, z15.s, z4.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "add z17.s, z17.s, z3.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -692,68 +863,56 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "1:\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "add z20.s, z20.s, z3.s\n"
+ "add z13.s, z13.s, z4.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -764,82 +923,67 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "1:\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -850,97 +994,79 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "1:\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z19.s, z19.s, z4.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -951,112 +1077,91 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z7.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8]\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
"addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "1:\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z19.s, z19.s, z4.s\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z20.s, z20.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z13.s, z13.s, z3.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z4.s\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -1067,127 +1172,103 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [%[outptr6]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z4.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z8.s, p0/z, [x8]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
"addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z7.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z4.s, p0/z, [%[outptr6], #2, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "1:\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "add z19.s, z19.s, z4.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z20.s, z20.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z13.s, z13.s, z3.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z4.s\n"
+ "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z15.s, z15.s, z2.s\n"
+ "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z16.s, z16.s, z3.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z4.s\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "st1w z15.s, p0, [%[outptr6]]\n"
+ "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -1199,141 +1280,114 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [%[outptr6]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr7]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8]\n"
- "ld1w z7.s, p0/z, [%[outptr7], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
"addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z6.s, p0/z, [%[outptr6], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "ld1w z7.s, p0/z, [%[outptr7], #2, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n"
- "addvl %[outptr7], %[outptr7], #3\n"
- "1:\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z19.s, z19.s, z4.s\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z20.s, z20.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z13.s, z13.s, z3.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z4.s\n"
+ "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z15.s, z15.s, z2.s\n"
+ "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z16.s, z16.s, z3.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z4.s\n"
+ "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z18.s, z18.s, z2.s\n"
+ "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n"
+ "add z19.s, z19.s, z3.s\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "add z20.s, z20.s, z4.s\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "st1w z15.s, p0, [%[outptr6]]\n"
+ "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "st1w z18.s, p0, [%[outptr7]]\n"
+ "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n"
+ "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n"
+ "addvl %[outptr7], %[outptr7], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp
index eb684e2..67a6eb3 100644
--- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp
@@ -26,11 +26,20 @@
#ifdef __ARM_FEATURE_SVE
template<>
-inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t alpha, const uint32_t beta)
+void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append)
{
- const uint32_t *inptr = in;
+ UNUSED(act);
- for (int y=y0; y<ymax; y+=8) {
+ const uint32_t *inptr = in;
+ uint32_t nullbias[192] = { 0 };
+
+ if (!append && !bias)
+ {
+ memset(nullbias, 0, (3 * get_vector_length<uint32_t>() * sizeof(uint32_t)));
+ }
+
+ for (int y=y0; y<ymax; y+=8)
+ {
uint32_t *outptr0 = out + (y * ldout) + x0;
uint32_t *outptr1 = outptr0 + ldout;
uint32_t *outptr2 = outptr1 + ldout;
@@ -42,42 +51,44 @@
const int height = ymax - y;
- for (int i=x0; i<xmax; i+=(3 * get_vector_length<uint32_t>())) {
- if (beta==0u)
+ for (int i=x0; i<xmax; i+=(3 * get_vector_length<uint32_t>()))
+ {
+ if (append)
{
- switch(height) {
+ switch(height)
+ {
case 1:
{
long w = xmax - i;
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "1:\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -88,41 +99,47 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "1:\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -133,50 +150,62 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -187,58 +216,76 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -249,67 +296,91 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -320,76 +391,106 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8]\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "1:\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -400,85 +501,121 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
- "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "1:\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[outptr6]]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
+ "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr6]]\n"
+ "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -490,93 +627,135 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[outptr0]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z10.s, p0/z, [%[inptr]]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[outptr1]]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z10.s, p0, [%[outptr0]]\n"
+ "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z8.s, p0/z, [%[outptr2]]\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p0, [%[outptr1]]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n"
- "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
- "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n"
- "addvl %[outptr7], %[outptr7], #3\n"
- "1:\n"
+ "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "ld1w z3.s, p0/z, [%[outptr3]]\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p0, [%[outptr2]]\n"
+ "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z11.s, z11.s, z3.s\n"
+ "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
+ "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[outptr4]]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
+ "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
+ "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z6.s\n"
+ "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "ld1w z9.s, p0/z, [%[outptr5]]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
+ "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
+ "ld1w z10.s, p1/z, [x8]\n"
+ "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z9.s\n"
+ "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z10.s, z10.s, z2.s\n"
+ "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "ld1w z4.s, p0/z, [%[outptr6]]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z11.s, z11.s, z3.s\n"
+ "st1w z17.s, p0, [%[outptr5]]\n"
+ "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
+ "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
+ "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "add z12.s, z12.s, z4.s\n"
+ "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
+ "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z13.s, z13.s, z5.s\n"
+ "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "ld1w z7.s, p0/z, [%[outptr7]]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "add z14.s, z14.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr6]]\n"
+ "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n"
+ "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n"
+ "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n"
+ "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "add z15.s, z15.s, z7.s\n"
+ "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n"
+ "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n"
+ "add z16.s, z16.s, z8.s\n"
+ "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "add z17.s, z17.s, z9.s\n"
+ "st1w z15.s, p0, [%[outptr7]]\n"
+ "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n"
+ "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n"
+ "addvl %[outptr7], %[outptr7], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -586,45 +765,46 @@
}
else
{
- switch(height) {
+ const uint32_t *biasptr = nullbias;
+ if (bias)
+ {
+ biasptr = bias + i;
+ }
+
+ switch(height)
+ {
case 1:
{
long w = xmax - i;
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
- "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "1:\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "add z14.s, z14.s, z3.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -635,53 +815,44 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "1:\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z15.s, z15.s, z4.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "add z17.s, z17.s, z3.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -692,68 +863,56 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "addvl %[outptr0], %[outptr0], #3\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "1:\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "addvl %[outptr0], %[outptr0], #3\n"
+ "add z20.s, z20.s, z3.s\n"
+ "add z13.s, z13.s, z4.s\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -764,82 +923,67 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "1:\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -850,97 +994,79 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
- "addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "1:\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
+ "addvl %[outptr1], %[outptr1], #3\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z19.s, z19.s, z4.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -951,112 +1077,91 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z7.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8]\n"
- "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
"addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "1:\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "add z19.s, z19.s, z4.s\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z20.s, z20.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z13.s, z13.s, z3.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z4.s\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -1067,127 +1172,103 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [%[outptr6]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z4.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z8.s, p0/z, [x8]\n"
- "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
"addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z7.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z4.s, p0/z, [%[outptr6], #2, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "1:\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "add z19.s, z19.s, z4.s\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z20.s, z20.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z13.s, z13.s, z3.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z4.s\n"
+ "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z15.s, z15.s, z2.s\n"
+ "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z16.s, z16.s, z3.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z4.s\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "st1w z15.s, p0, [%[outptr6]]\n"
+ "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
@@ -1199,141 +1280,114 @@
long p = 0;
/* Optimized routine to copy an entire block */
__asm __volatile (
- "mov z2.s, %s[alpha]\n"
"addvl x8, %[inptr], #16\n"
- "mov z3.s, %s[beta]\n"
"whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
"incw %[p], all, mul #1\n"
"prfm PLDL1KEEP, [%[inptr], #0x180]\n"
- "ld1w z4.s, p0/z, [%[outptr0]]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
- "ld1w z8.s, p0/z, [%[inptr]]\n"
- "ld1w z5.s, p0/z, [%[outptr1]]\n"
- "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4]]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0]]\n"
- "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5]]\n"
- "ld1w z6.s, p0/z, [%[outptr6]]\n"
- "st1w z9.s, p0, [%[outptr1]]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr7]]\n"
- "st1w z10.s, p0, [%[outptr2]]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3]]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4]]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr5]]\n"
- "st1w z10.s, p0, [%[outptr6]]\n"
- "st1w z11.s, p0, [%[outptr7]]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "ld1w z2.s, p0/z, [%[biasptr]]\n"
+ "whilelt p1.s, %[p], %[w]\n"
+ "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n"
"incw %[p], all, mul #1\n"
+ "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n"
- "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
- "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n"
- "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n"
- "ld1w z4.s, p0/z, [%[outptr4], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "ld1w z5.s, p0/z, [%[outptr5], #1, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr6], #1, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [x8]\n"
- "ld1w z7.s, p0/z, [%[outptr7], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n"
- "add z9.s, z9.s, z5.s\n"
- "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n"
- "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n"
- "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n"
- "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n"
- "whilelt p0.s, %[p], %[w]\n"
- "b.none 1f\n"
- "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
- "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
- "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n"
+ "ld1w z13.s, p0/z, [%[inptr]]\n"
+ "whilelt p2.s, %[p], %[w]\n"
+ "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "add z13.s, z13.s, z2.s\n"
+ "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
+ "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x200]\n"
- "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
- "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n"
- "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
- "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n"
- "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n"
+ "add z14.s, z14.s, z3.s\n"
+ "st1w z13.s, p0, [%[outptr0]]\n"
+ "add z15.s, z15.s, z4.s\n"
+ "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
+ "add z16.s, z16.s, z2.s\n"
+ "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
+ "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "add z17.s, z17.s, z3.s\n"
+ "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
+ "add z19.s, z19.s, z2.s\n"
+ "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
+ "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "add z20.s, z20.s, z3.s\n"
+ "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
+ "add z14.s, z14.s, z2.s\n"
"addvl %[outptr0], %[outptr0], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n"
- "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n"
+ "st1w z16.s, p0, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "add z15.s, z15.s, z3.s\n"
+ "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
"addvl %[outptr1], %[outptr1], #3\n"
- "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n"
+ "add z17.s, z17.s, z2.s\n"
+ "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
"prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n"
- "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n"
- "addvl %[outptr2], %[outptr2], #3\n"
- "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
- "add z8.s, z8.s, z4.s\n"
- "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n"
- "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n"
- "addvl %[outptr3], %[outptr3], #3\n"
- "ld1w z6.s, p0/z, [%[outptr6], #2, MUL VL]\n"
- "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
- "add z9.s, z9.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n"
- "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n"
- "addvl %[outptr4], %[outptr4], #3\n"
- "ld1w z7.s, p0/z, [%[outptr7], #2, MUL VL]\n"
- "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n"
- "addvl %[outptr5], %[outptr5], #3\n"
- "add z10.s, z10.s, z6.s\n"
- "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n"
- "add z11.s, z11.s, z7.s\n"
- "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n"
- "addvl %[outptr6], %[outptr6], #3\n"
- "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n"
- "addvl %[outptr7], %[outptr7], #3\n"
- "1:\n"
+ "st1w z19.s, p0, [%[outptr2]]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "add z18.s, z18.s, z3.s\n"
+ "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
"addvl %[inptr], %[inptr], #24\n"
+ "add z19.s, z19.s, z4.s\n"
+ "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
+ "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
+ "addvl %[outptr2], %[outptr2], #3\n"
+ "add z20.s, z20.s, z2.s\n"
+ "ld1w z13.s, p1/z, [x8]\n"
+ "st1w z14.s, p0, [%[outptr3]]\n"
+ "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
+ "add z13.s, z13.s, z3.s\n"
+ "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
+ "add z14.s, z14.s, z4.s\n"
+ "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
+ "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
+ "addvl %[outptr3], %[outptr3], #3\n"
+ "add z15.s, z15.s, z2.s\n"
+ "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
+ "st1w z17.s, p0, [%[outptr4]]\n"
+ "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
+ "add z16.s, z16.s, z3.s\n"
+ "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
+ "add z17.s, z17.s, z4.s\n"
+ "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n"
+ "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
+ "addvl %[outptr4], %[outptr4], #3\n"
+ "add z18.s, z18.s, z2.s\n"
+ "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n"
+ "st1w z20.s, p0, [%[outptr5]]\n"
+ "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n"
+ "add z19.s, z19.s, z3.s\n"
+ "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
+ "add z20.s, z20.s, z4.s\n"
+ "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
+ "addvl %[outptr5], %[outptr5], #3\n"
+ "st1w z15.s, p0, [%[outptr6]]\n"
+ "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
+ "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
+ "addvl %[outptr6], %[outptr6], #3\n"
+ "st1w z18.s, p0, [%[outptr7]]\n"
+ "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n"
+ "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n"
+ "addvl %[outptr7], %[outptr7], #3\n"
: [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
[inptr] "+r" (inptr), [p] "+r" (p)
- : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w)
- : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc"
+ : [w] "r" (w), [biasptr] "r" (biasptr)
+ : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
);
}
break;
diff --git a/src/core/NEON/kernels/arm_gemm/misc.cpp b/src/core/NEON/kernels/arm_gemm/misc.cpp
index 7b345e2..6758a88 100644
--- a/src/core/NEON/kernels/arm_gemm/misc.cpp
+++ b/src/core/NEON/kernels/arm_gemm/misc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
diff --git a/src/core/NEON/kernels/arm_gemm/profiler.hpp b/src/core/NEON/kernels/arm_gemm/profiler.hpp
deleted file mode 100644
index 1b944c4..0000000
--- a/src/core/NEON/kernels/arm_gemm/profiler.hpp
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright (c) 2017-2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef CYCLE_PROFILING
-
-#include "../perf.h"
-
-#ifndef NO_MULTI_THREADING
-#include <mutex>
-#endif
-
-namespace arm_gemm {
-
-#ifndef NO_MULTI_THREADING
-extern std::mutex report_mutex;
-#endif
-
-class profiler {
-private:
- static const int maxevents = 100000;
- unsigned long times[maxevents] = { };
- unsigned long units[maxevents] = { };
- int events[maxevents] = { };
- int currentevent=0;
- int countfd=0;
-
- class ScopedProfilerClass {
- private:
- profiler &_parent;
- bool legal=false;
-
- public:
- ScopedProfilerClass(profiler &prof, int i, unsigned long u) : _parent(prof) {
- if (prof.currentevent==maxevents)
- return;
-
- prof.events[prof.currentevent]=i;
- prof.units[prof.currentevent]=u;
- legal=true;
- start_counter(prof.countfd);
- }
-
- ~ScopedProfilerClass() {
- if (!legal) return;
-
- long long cycs = stop_counter(_parent.countfd);
- _parent.times[_parent.currentevent++] = cycs;
- }
- };
-
-public:
- profiler() {
- countfd=open_cycle_counter();
- }
-
- ~profiler() {
- close(countfd);
- int tots[5];
- unsigned long counts[5];
- unsigned long tunits[5];
- const char * descs[] = { "Prepare A", "Prepare B", "Kernel", "Merge" };
-
- for (int i=1; i<5; i++) {
- tots[i] = 0;
- counts[i] = 0;
- tunits[i] = 0;
- }
-
- for (int i=0; i<currentevent; i++) {
-// printf("%10s: %ld\n", descs[events[i]-1], times[i]);
- tots[events[i]]++;
- counts[events[i]] += times[i];
- tunits[events[i]] += units[i];
- }
-
-#ifdef NO_MULTI_THREADING
- printf("Profiled events:\n");
-#else
- std::lock_guard<std::mutex> lock(report_mutex);
- printf("Profiled events (cpu %d):\n", sched_getcpu());
-#endif
-
- printf("%20s %9s %9s %9s %12s %9s\n", "", "Events", "Total", "Average", "Bytes/MACs", "Per cycle");
- for (int i=1; i<5; i++) {
- printf("%20s: %9d %9ld %9ld %12lu %9.2f\n",descs[i-1],tots[i],counts[i],counts[i]/tots[i],tunits[i],(float)tunits[i]/counts[i]);
- }
- }
-
- template <typename T>
- void operator() (int i, unsigned long u, T func) {
- if (currentevent==maxevents) {
- func();
- } else {
- events[currentevent] = i;
- units[currentevent] = u;
- start_counter(countfd);
- func();
- long long cycs = stop_counter(countfd);
- times[currentevent++] = cycs;
- }
- }
-
- ScopedProfilerClass ScopedProfiler(int i, unsigned long u) {
- return ScopedProfilerClass(*this, i, u);
- }
-};
-
-#endif // CYCLE_PROFILING
-
-} // namespace arm_gemm
-
-#define PROFILE_PREPA 1
-#define PROFILE_PREPB 2
-#define PROFILE_KERNEL 3
-#define PROFILE_MERGE 4
diff --git a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp
index 5e67bc9..188dd0b 100644
--- a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp
+++ b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp
@@ -41,7 +41,7 @@
int32_t *_row_sums = nullptr;
int32_t *_col_sums = nullptr;
ARequantizeLayer32 _params;
- GemmArgs<Tr> _args;
+ GemmArgs _args;
barrier _barrier;
void *working_space = nullptr;
@@ -80,12 +80,13 @@
/* Use the first part of our working space for the subgemm result, pass the operand details straight through. */
_subgemm->set_arrays(this->_Aptr, this->_lda, this->_A_batch_stride, this->_A_multi_stride,
this->_Bptr, this->_ldb, this->_B_multi_stride,
- reinterpret_cast<Tgemm *>(working_space), _args._Nsize, (_args._Nsize * _args._Msize), (_args._Nsize * _args._Msize * _args._nbatches));
+ reinterpret_cast<Tgemm *>(working_space), _args._Nsize, (_args._Nsize * _args._Msize), (_args._Nsize * _args._Msize * _args._nbatches),
+ nullptr, 0);
}
void col_sums_pretransposed(const To *B, const int ldb, const int B_multi_stride) {
for (unsigned int multi=0; multi<_args._nmulti; multi++) {
- compute_col_sums(_params, _args._Nsize, _args._Ksize, B + (multi * B_multi_stride), ldb, _col_sums + (multi * _args._Nsize), _args._Ksize, 0);
+ compute_col_sums(_params, _args._Nsize, _args._Ksize, B + (multi * B_multi_stride), ldb, _col_sums + (multi * _args._Nsize), _args._Ksize, multi, 0);
}
}
@@ -94,7 +95,7 @@
unsigned int last_col = ((threadid + 1) * _args._Nsize) / _args._maxthreads;
for (unsigned int multi=0; multi<_args._nmulti; multi++) {
- compute_col_sums(_params, (last_col - first_col), _args._Ksize, this->_Bptr + (multi * this->_B_multi_stride) + first_col, this->_ldb, _col_sums + (multi * _args._Nsize) + first_col, _args._Ksize, first_col);
+ compute_col_sums(_params, (last_col - first_col), _args._Ksize, this->_Bptr + (multi * this->_B_multi_stride) + first_col, this->_ldb, _col_sums + (multi * _args._Nsize) + first_col, _args._Ksize, multi, first_col);
}
}
@@ -121,8 +122,11 @@
public:
- QuantizeWrapper(const GemmArgs<Tr> &args, const ARequantizeLayer32 &qp) : _params(qp), _args(args), _barrier(args._maxthreads) {
- GemmArgs<Tgemm> newargs = GemmArgs<Tgemm>(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._trA, args._trB, 1, 0, args._maxthreads, args._pretransposed_hint, nullptr);
+ QuantizeWrapper(const QuantizeWrapper &) = delete;
+ QuantizeWrapper operator=(const QuantizeWrapper &) = delete;
+
+ QuantizeWrapper(const GemmArgs &args, const ARequantizeLayer32 &qp) : _params(qp), _args(args), _barrier(args._maxthreads) {
+ GemmArgs newargs = GemmArgs(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._trA, args._trB, Activation(), args._maxthreads, args._pretransposed_hint, nullptr);
_subgemm = gemm<To, Tgemm>(newargs);
if (_subgemm == nullptr) {
@@ -134,15 +138,11 @@
}
}
- QuantizeWrapper(const QuantizeWrapper &) = delete;
- QuantizeWrapper &operator=(const QuantizeWrapper &) = delete;
- QuantizeWrapper(QuantizeWrapper &&) = default;
- QuantizeWrapper &operator=(QuantizeWrapper &&) = default;
-
void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride,
const To *B, const int ldb, const int B_multi_stride,
- Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride) override {
- GemmCommon<To, Tr>::set_arrays(A, lda, A_batch_stride, A_multi_stride, B, ldb, B_multi_stride, C, ldc, C_batch_stride, C_multi_stride);
+ Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride,
+ const Tr *bias, const int bias_multi_stride) override {
+ GemmCommon<To, Tr>::set_arrays(A, lda, A_batch_stride, A_multi_stride, B, ldb, B_multi_stride, C, ldc, C_batch_stride, C_multi_stride, bias, bias_multi_stride);
arrays_set = true;
set_child_arrays();
@@ -232,8 +232,9 @@
_col_sums = reinterpret_cast<int32_t *>(buffer);
}
- void set_quantized_bias(const int32_t *bias) override {
+ void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override {
_params.bias = bias;
+ _params.bias_multi_stride = bias_multi_stride;
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/quantized.cpp b/src/core/NEON/kernels/arm_gemm/quantized.cpp
index 28f01bd..bffb7dd 100644
--- a/src/core/NEON/kernels/arm_gemm/quantized.cpp
+++ b/src/core/NEON/kernels/arm_gemm/quantized.cpp
@@ -495,10 +495,10 @@
* We could do 64 adds in the signed case, but that
* optimization is not worth the complexity.
*/
- if (i > 0 && ((i & 31) == 0)) {
- finalsums[r] = vpadalq_s16(finalsums[r], sums[r]);
- sums[r] = vdupq_n_s16(0);
- }
+ if (i > 0 && ((i & 31) == 0)) {
+ finalsums[r] = vpadalq_s16(finalsums[r], sums[r]);
+ sums[r] = vdupq_n_s16(0);
+ }
sums[r] = accumulate_16(input + (r * in_stride) + (i * 16), sums[r]);
}
}
@@ -526,6 +526,7 @@
* that the terms can simply be added in the requantize code.
* */
switch (rows) {
+ default:
case 1:
/* If we only have one output, just use ADDV. Multiply
* the offset into all four components separately so it
@@ -567,8 +568,6 @@
vst1q_s32(row_bias, t0);
break;
- default:
- break;
}
}
@@ -736,12 +735,11 @@
}
}
-
/* "first_col" parameter is used to offset the read into the qp.bias array,
* in cases where we are not computing the first columns of the output (i.e.
* in multithreaded cases where we divide columns across threads) */
template<typename T>
-void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int first_col) {
+void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col) {
memset(reinterpret_cast<void *>(col_bias), 0, width * sizeof(int32_t));
for (unsigned int row=0; row<height; row+=4) {
@@ -752,6 +750,7 @@
if (numcols==16) {
switch(numrows) {
+ default:
case 1:
add_block<1>(input + row * in_stride + col, in_stride, col_bias + col);
break;
@@ -767,8 +766,6 @@
case 4:
add_block<4>(input + row * in_stride + col, in_stride, col_bias + col);
break;
- default:
- break;
}
} else {
for (; col<width; col++) {
@@ -788,15 +785,15 @@
result = (qp.a_offset * qp.b_offset * depth) - (result * qp.a_offset);
if (qp.bias != nullptr) {
- result += qp.bias[col + first_col];
+ result += qp.bias[multi * qp.bias_multi_stride + col + first_col];
}
col_bias[col] = result;
}
}
-template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const int8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int first_col);
-template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const uint8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int first_col);
+template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const int8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col);
+template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const uint8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/quantized.hpp b/src/core/NEON/kernels/arm_gemm/quantized.hpp
index eaa303b..a227507 100644
--- a/src/core/NEON/kernels/arm_gemm/quantized.hpp
+++ b/src/core/NEON/kernels/arm_gemm/quantized.hpp
@@ -37,6 +37,6 @@
template<typename T>
void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height,
const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth,
- unsigned int first_col);
+ unsigned int multi, unsigned int first_col);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp
index 2c661e7..c3c1e8d 100644
--- a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp
+++ b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp
@@ -64,8 +64,8 @@
}
template<typename TOut>
- void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut alpha, const TOut beta) {
- MergeResults<width, height>(out, in, stride, y0, ymax, x0, xmax, alpha, beta);
+ void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut *bias, const Activation act, bool append) {
+ MergeResults<width, height>(out, in, stride, y0, ymax, x0, xmax, bias, act, append);
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp
index b7323eb..6b64e5e 100644
--- a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp
+++ b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp
@@ -63,8 +63,8 @@
}
template<typename TOut>
- void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut alpha, const TOut beta) {
- MergeResults<width_vectors / mmla, height, true>(out, in, stride, y0, ymax, x0, xmax, alpha, beta);
+ void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut *bias, const Activation act, bool append) {
+ MergeResults<width_vectors / mmla, height, true>(out, in, stride, y0, ymax, x0, xmax, bias, act, append);
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/transform.hpp b/src/core/NEON/kernels/arm_gemm/transform.hpp
index d790d56..bdae903 100644
--- a/src/core/NEON/kernels/arm_gemm/transform.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transform.hpp
@@ -84,7 +84,6 @@
*out++ = static_cast<TOut>(0);
}
}
-
// "row" tail - row is out of range so fill with zeros always.
TOut zeroval = static_cast<TOut>(0);
int pads = blank_rows * (fill_cols + blank_cols);